How to load a learning rate scheduler state dict?
I have a model and a learning rate scheduler. I'm saving the model and optimizer using the state dict method that is shown here.
import torch
import torch.nn as nn
import torch.optim as optim

class net_x(nn.Module): 
        def __init__(self):
            super(net_x, self).__init__()
            self.fc1=nn.Linear(2, 20) 
            self.fc2=nn.Linear(20, 20)
            self.out=nn.Linear(20, 4) 

        def forward(self, x):
            x=self.fc1(x)
            x=self.fc2(x)
            x=self.out(x)
            return x

nx = net_x()


r = torch.tensor([1.0,2.0])
optimizer = optim.Adam(nx.parameters(), lr = 0.1)
scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=1e-2, max_lr=0.1, step_size_up=1, mode=&quot;triangular2&quot;, cycle_momentum=False)

path = 'opt.pt'
for epoch in range(10):
    optimizer.zero_grad()
    net_predictions = nx(r)
    loss = torch.sum(torch.randint(0,10,(4,)) - net_predictions)
    loss.backward()
    optimizer.step()
    scheduler.step()
    print('loss:' , loss)
    torch.save({    'epoch': epoch,
                    'net_x_state_dict': nx.state_dict(),
                    'optimizer_state_dict': optimizer.state_dict(),
                    'scheduler': scheduler,
                    }, path)

PATH = control_path
checkpoint = torch.load(path)        
nx.load_state_dict(checkpoint['net_x_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
scheduler.load_state_dict(checkpoint['scheduler'])

The code runs just fine without the part that I'm loading the scheduler state dict, so I'm not sure what I'm doing wrong. I'm trying to load the state dict as mentioned here, but I'm getting this error:
TypeError                                 Traceback (most recent call last)
&lt;ipython-input-7-e3217d6dd870&gt; in &lt;module&gt;
     42 nx.load_state_dict(checkpoint['net_x_state_dict'])
     43 optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
---&gt; 44 scheduler.load_state_dict(checkpoint['scheduler'])

~/anaconda3/lib/python3.7/site-packages/torch/optim/lr_scheduler.py in load_state_dict(self, state_dict)
     92                 from a call to :meth:`state_dict`.
     93         &quot;&quot;&quot;
---&gt; 94         self.__dict__.update(state_dict)
     95 
     96     def get_last_lr(self):

TypeError: 'CyclicLR' object is not iterable


Since we have to extract the state_dict() values from scheduler before saving i.e in torch.save() method
Below code will work
import torch
import torch.nn as nn
import torch.optim as optim

class net_x(nn.Module): 
        def __init__(self):
            super(net_x, self).__init__()
            self.fc1=nn.Linear(2, 20) 
            self.fc2=nn.Linear(20, 20)
            self.out=nn.Linear(20, 4) 

        def forward(self, x):
            x=self.fc1(x)
            x=self.fc2(x)
            x=self.out(x)
            return x

nx = net_x()


r = torch.tensor([1.0,2.0])
optimizer = optim.Adam(nx.parameters(), lr = 0.1)
scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=1e-2, max_lr=0.1, step_size_up=1, mode=&quot;triangular2&quot;, cycle_momentum=False)

path = 'opt.pt'
for epoch in range(10):
    optimizer.zero_grad()
    net_predictions = nx(r)
    loss = torch.sum(torch.randint(0,10,(4,)) - net_predictions)
    loss.backward()
    optimizer.step()
    scheduler.step()
    print('loss:' , loss)
    torch.save({    'epoch': epoch,
                    'net_x_state_dict': nx.state_dict(),
                    'optimizer_state_dict': optimizer.state_dict(),
                    'scheduler': scheduler.state_dict(),    # HERE IS THE CHANGE
                    }, path)

PATH = control_path
checkpoint = torch.load(path)        
nx.load_state_dict(checkpoint['net_x_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
scheduler.load_state_dict(checkpoint['scheduler'])

