Update imagic.py

imagic.py

@@ -318,7 +318,9 @@ class ImagicStableDiffusionPipeline(DiffusionPipeline):
 
                 with torch.no_grad():
                     torch.cuda.empty_cache()
-                optimizer.step()
+
+                with torch.no_grad():
+                    optimizer.step()
                 optimizer.zero_grad()
 
             # Checks if the accelerator has performed an optimization step behind the scenes