wilds-ijepa/src/utils/logging.py

# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
#

import os
import torch


def gpu_timer(closure, log_timings=True):
    """ Helper to time gpu-time to execute closure() """
    log_timings = log_timings and torch.cuda.is_available()

    elapsed_time = -1.
    if log_timings:
        start = torch.cuda.Event(enable_timing=True)
        end = torch.cuda.Event(enable_timing=True)
        start.record()

    result = closure()

    if log_timings:
        end.record()
        torch.cuda.synchronize()
        elapsed_time = start.elapsed_time(end)

    return result, elapsed_time


def _format_value(value):
    if value is None:
        return None
    if isinstance(value, float):
        return f"{value:g}"
    if isinstance(value, (list, tuple)):
        return "-".join(_format_value(v) for v in value)
    if isinstance(value, bool):
        return "1" if value else "0"
    return str(value)


def build_run_name(args):
    meta_args = args.get("meta", {})
    data_args = args.get("data", {})
    opt_args = args.get("optimization", {})
    mask_args = args.get("mask", {})
    val_args = args.get("validation", {})

    model_name = meta_args.get("model_name", "model")
    representation_type = meta_args.get("representation_type")
    head_type = meta_args.get("head_type")
    patch_size = mask_args.get("patch_size", meta_args.get("patch_size"))
    crop_size = data_args.get("crop_size", meta_args.get("crop_size"))
    batch_size = data_args.get("batch_size")
    optimizer = opt_args.get("optimizer", "opt")
    lr = opt_args.get("lr")
    weight_decay = opt_args.get("weight_decay")
    epochs = opt_args.get("epochs")

    parts = [model_name]

    def add(prefix, value):
        formatted = _format_value(value)
        if formatted is not None:
            parts.append(f"{prefix}{formatted}")

    add("p", patch_size)
    add("c", crop_size)
    add("bs", batch_size)
    add("rep", representation_type)
    add("head", head_type)
    parts.append(str(optimizer).lower())
    add("lr", lr)
    add("wd", weight_decay)
    add("ep", epochs)
    add("sched", opt_args.get("lr_schedule"))
    add("ms", opt_args.get("step_milestones"))
    add("sg", opt_args.get("step_gamma"))
    add("wu", opt_args.get("warmup"))
    add("mom", opt_args.get("momentum"))
    add("leta", opt_args.get("lars_eta"))
    add("leps", opt_args.get("lars_eps"))
    add("ipe", opt_args.get("ipe_scale"))
    add("cs", data_args.get("crop_scale"))
    add("eval", val_args.get("eval_every"))

    return "-".join([p for p in parts if p])


def _extract_run_name_from_checkpoint(meta_args):
    checkpoint_path = meta_args.get("checkpoint_path")
    if checkpoint_path:
        folder = os.path.dirname(checkpoint_path)
    else:
        folder = meta_args.get("checkpoint_folder")
    if not folder:
        return None
    return os.path.basename(os.path.normpath(folder))


def resolve_log_dir(args, stage="train"):
    log_args = args.setdefault("logging", {})
    auto_folder = log_args.get("auto_folder", True)
    if auto_folder or not log_args.get("folder"):
        run_name = log_args.get("run_name")
        if stage == "eval" and not run_name:
            meta_args = args.get("meta", {})
            run_name = _extract_run_name_from_checkpoint(meta_args)
        if not run_name:
            run_name = build_run_name(args)
        base_dir = "experiment_logs"
        if stage == "eval":
            folder = os.path.join(base_dir, "eval-wilds", run_name)
        else:
            folder = os.path.join(base_dir, run_name)
        log_args["folder"] = folder
    os.makedirs(log_args["folder"], exist_ok=True)
    return log_args["folder"]


class CSVLogger(object):

    def __init__(self, fname, *argv):
        self.fname = fname
        self.types = []
        # -- print headers
        with open(self.fname, '+a') as f:
            for i, v in enumerate(argv, 1):
                self.types.append(v[0])
                if i < len(argv):
                    print(v[1], end=',', file=f)
                else:
                    print(v[1], end='\n', file=f)

    def log(self, *argv):
        with open(self.fname, '+a') as f:
            for i, tv in enumerate(zip(self.types, argv), 1):
                end = ',' if i < len(argv) else '\n'
                print(tv[0] % tv[1], end=end, file=f)


class AverageMeter(object):
    """computes and stores the average and current value"""

    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.max = float('-inf')
        self.min = float('inf')
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        try:
            self.max = max(val, self.max)
            self.min = min(val, self.min)
        except Exception:
            pass
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count


def grad_logger(named_params):
    stats = AverageMeter()
    stats.first_layer = None
    stats.last_layer = None
    for n, p in named_params:
        if (p.grad is not None) and not (n.endswith('.bias') or len(p.shape) == 1):
            grad_norm = float(torch.norm(p.grad.data))
            stats.update(grad_norm)
            if 'qkv' in n:
                stats.last_layer = grad_norm
                if stats.first_layer is None:
                    stats.first_layer = grad_norm
    if stats.first_layer is None or stats.last_layer is None:
        stats.first_layer = stats.last_layer = 0.
    return stats