Skip to content

unitorch.cli.schedulers¤

CosineWarmupScheduler¤

Tip

core/scheduler/cosine_warmup is the section for configuration of CosineWarmupScheduler.

Bases: CosineWarmupScheduler, SchedulerCheckpointMixin

Source code in src/unitorch/cli/schedulers/warmup.py 
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
def __init__(
    self,
    optimizer: optim.Optimizer,
    num_warmup_steps: int,
    num_training_steps: int,
    num_cycles: Optional[float] = 0.5,
    last_epoch: Optional[int] = -1,
):
    super().__init__(
        optimizer=optimizer,
        num_warmup_steps=num_warmup_steps,
        num_training_steps=num_training_steps,
        num_cycles=num_cycles,
        last_epoch=last_epoch,
    )

from_config classmethod ¤

from_config(config, **kwargs)
Source code in src/unitorch/cli/schedulers/warmup.py 
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
@classmethod
@config_defaults_init("core/scheduler/cosine_warmup")
def from_config(cls, config, **kwargs):
    num_warmup_steps = config.getdefault(
        "core/scheduler/cosine_warmup", "num_warmup_steps", -1
    )
    num_warmup_rate = config.getdefault(
        "core/scheduler/cosine_warmup", "num_warmup_rate", 0.001
    )
    num_cycles = config.getdefault(
        "core/scheduler/cosine_warmup", "num_cycles", 0.5
    )
    num_training_steps = kwargs.get("num_training_steps", 1000000)
    if num_warmup_steps < 0:
        num_warmup_steps = int(num_training_steps * num_warmup_rate)
    return {
        "num_warmup_steps": num_warmup_steps,
        "num_training_steps": num_training_steps,
        "num_cycles": num_cycles,
    }

LinearWarmupScheduler¤

Tip

core/scheduler/linear_warmup is the section for configuration of LinearWarmupScheduler.

Bases: LinearWarmupScheduler, SchedulerCheckpointMixin

Source code in src/unitorch/cli/schedulers/warmup.py 
56
57
58
59
60
61
62
63
64
65
66
67
68
def __init__(
    self,
    optimizer: optim.Optimizer,
    num_warmup_steps: int,
    num_training_steps: int,
    last_epoch: Optional[int] = -1,
):
    super().__init__(
        optimizer=optimizer,
        num_warmup_steps=num_warmup_steps,
        num_training_steps=num_training_steps,
        last_epoch=last_epoch,
    )

from_config classmethod ¤

from_config(config, **kwargs)
Source code in src/unitorch/cli/schedulers/warmup.py 
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
@classmethod
@config_defaults_init("core/scheduler/linear_warmup")
def from_config(cls, config, **kwargs):
    num_warmup_steps = config.getdefault(
        "core/scheduler/linear_warmup", "num_warmup_steps", -1
    )
    num_warmup_rate = config.getdefault(
        "core/scheduler/linear_warmup", "num_warmup_rate", 0.001
    )
    num_training_steps = kwargs.get("num_training_steps", 1000000)
    if num_warmup_steps < 0:
        num_warmup_steps = int(num_training_steps * num_warmup_rate)
    return {
        "num_warmup_steps": num_warmup_steps,
        "num_training_steps": num_training_steps,
    }