|
|
|
@ -26,23 +26,23 @@ class CosineLRScheduler(Scheduler):
|
|
|
|
|
k-decay option based on `k-decay: A New Method For Learning Rate Schedule` - https://arxiv.org/abs/2004.05909
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
optimizer (torch.optim.Optimizer): torch optimizer to schedule
|
|
|
|
|
t_initial (int): Number of epochs it initial (first) cycle.
|
|
|
|
|
lr_min (float, optional): Minimum learning rate to use during the scheduling. Defaults to 0..
|
|
|
|
|
cycle_mul (float, optional): Multiplyer for cycle length. Defaults to 1..
|
|
|
|
|
cycle_decay (float, optional): Factor to decay lr at next cycle. Defaults to 1..
|
|
|
|
|
cycle_limit (int, optional): Number of cycles. Defaults to 1.
|
|
|
|
|
warmup_t (int, optional): Number of epochs to warmup. Defaults to 0.
|
|
|
|
|
warmup_lr_init (float, optional): Initial learning rate during warmup . Defaults to 0.
|
|
|
|
|
warmup_prefix (bool, optional): If True, after warmup annealing starts from initial LR. Defaults to False.
|
|
|
|
|
t_in_epochs (bool, optional): If set to False, returned lr are None. Defaults to True.
|
|
|
|
|
noise_range_t (Union[int, float, List[int, float]], optional): Epoch when noise starts.\
|
|
|
|
|
If list or tuple - epoch range, when noise applied. Defaults to None.
|
|
|
|
|
noise_pct (float, optional): Percentage of noise to add. Defaults to 0.67.
|
|
|
|
|
noise_std (float, optional): Noise standard deviation. Defaults to 1.0.
|
|
|
|
|
noise_seed (int, optional): Seed to use to add random noise. Defaults to 42.
|
|
|
|
|
k_decay (float, optional): Power for k_decay. Defaults to 1.0.
|
|
|
|
|
initialize (bool, optional): Add initial_{field_name} to optimizer param group. Defaults to True.
|
|
|
|
|
optimizer (torch.optim.Optimizer): torch optimizer to schedule
|
|
|
|
|
t_initial (int): Number of epochs it initial (first) cycle.
|
|
|
|
|
lr_min (float, optional): Minimum learning rate to use during the scheduling. Defaults to 0..
|
|
|
|
|
cycle_mul (float, optional): Multiplyer for cycle length. Defaults to 1..
|
|
|
|
|
cycle_decay (float, optional): Factor to decay lr at next cycle. Defaults to 1..
|
|
|
|
|
cycle_limit (int, optional): Number of cycles. Defaults to 1.
|
|
|
|
|
warmup_t (int, optional): Number of epochs to warmup. Defaults to 0.
|
|
|
|
|
warmup_lr_init (float, optional): Initial learning rate during warmup . Defaults to 0.
|
|
|
|
|
warmup_prefix (bool, optional): If True, after warmup annealing starts from initial LR. Defaults to False.
|
|
|
|
|
t_in_epochs (bool, optional): If set to False, returned lr are None. Defaults to True.
|
|
|
|
|
noise_range_t (Union[int, float, List[int | float]], optional): Epoch when noise starts.\
|
|
|
|
|
If list or tuple - epoch range, when noise applied. Defaults to None.
|
|
|
|
|
noise_pct (float, optional): Percentage of noise to add. Defaults to 0.67.
|
|
|
|
|
noise_std (float, optional): Noise standard deviation. Defaults to 1.0.
|
|
|
|
|
noise_seed (int, optional): Seed to use to add random noise. Defaults to 42.
|
|
|
|
|
k_decay (float, optional): Power for k_decay. Defaults to 1.0.
|
|
|
|
|
initialize (bool, optional): Add initial_{field_name} to optimizer param group. Defaults to True.
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
def __init__(self,
|
|
|
|
@ -56,7 +56,7 @@ class CosineLRScheduler(Scheduler):
|
|
|
|
|
warmup_lr_init: float = 0,
|
|
|
|
|
warmup_prefix: bool = False,
|
|
|
|
|
t_in_epochs: bool = True,
|
|
|
|
|
noise_range_t: Union[int, float, List[int, float]] = None,
|
|
|
|
|
noise_range_t: Union[int, float, List[Union[int, float]]] = None,
|
|
|
|
|
noise_pct: float = 0.67,
|
|
|
|
|
noise_std: float = 1.0,
|
|
|
|
|
noise_seed: int = 42,
|
|
|
|
|