<p dir="auto">Memory keeps growing at every iteration. My pytorch version 0.4.0a0+4d405a4.</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="import torch
from torch.autograd import grad, Variable
from torchvision import models

model = models.resnet50().cuda()
for k in range(20):
    x   = Variable(torch.rand(8, 3, 224, 224).cuda(), requires_grad=True)
    dx, = grad(model(x).sum(), x, create_graph=True)
    y = model(x + dx).sum()
    y.backward()"><pre class="notranslate"><code class="notranslate">import torch
from torch.autograd import grad, Variable
from torchvision import models

model = models.resnet50().cuda()
for k in range(20):
    x   = Variable(torch.rand(8, 3, 224, 224).cuda(), requires_grad=True)
    dx, = grad(model(x).sum(), x, create_graph=True)
    y = model(x + dx).sum()
    y.backward()
</code></pre></div>
<p dir="auto">This code is from <a href="https://github.com/pytorch/pytorch/issues/2361" data-hovercard-type="issue" data-hovercard-url="/pytorch/pytorch/issues/2361/hovercard">Double backward memory leak</a>. It work well and no memory leak. But use</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="y = dx.mean()
y.backward()"><pre class="notranslate"><code class="notranslate">y = dx.mean()
y.backward()
</code></pre></div>
<p dir="auto">the memory leak is happen.</p>
<p dir="auto">Originally reported by <a class="user-mention notranslate" data-hovercard-type="user" data-hovercard-url="/users/dmarnerides/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="https://github.com/dmarnerides">@dmarnerides</a> in <a class="issue-link js-issue-link" data-error-text="Failed to load title" data-id="274653738" data-permission-text="Title is private" data-url="https://github.com/pytorch/pytorch/issues/3743" data-hovercard-type="issue" data-hovercard-url="/pytorch/pytorch/issues/3743/hovercard" href="https://github.com/pytorch/pytorch/issues/3743">#3743</a>:</p>
<hr>
<p dir="auto">I can reproduce the memory leak with this:</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="import gc
import resource
import torch
from torch import nn, autograd
from torch.autograd import Variable

class Network(nn.Module):
    def __init__(self):
        super(Network, self).__init__()
        self.main = nn.Sequential(
            nn.Conv2d(1, 1, 1, 1),
            nn.Conv2d(1, 1, 1, 1),
        )
    def forward(self, v_x):
        return self.main(v_x).view(v_x.size(0), 1)

net = Network()

i = 0
while True:
    v_in = Variable(torch.Tensor(2,1,1,1), requires_grad=True)
    grad_out = Variable(torch.ones(2,1,1,1))

    gradient = autograd.grad(outputs=net(v_in), inputs=v_in,
                             grad_outputs=grad_out,
                             create_graph=True, retain_graph=True, 
                             only_inputs=True)[0]
    gradient.mean().backward()

    i += 1
    if i % 512 == 0:
        gc.collect()
        max_mem_used = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
        print(&quot;{:.2f} MB&quot;.format(max_mem_used / 1024))"><pre class="notranslate"><code class="notranslate">import gc
import resource
import torch
from torch import nn, autograd
from torch.autograd import Variable

class Network(nn.Module):
    def __init__(self):
        super(Network, self).__init__()
        self.main = nn.Sequential(
            nn.Conv2d(1, 1, 1, 1),
            nn.Conv2d(1, 1, 1, 1),
        )
    def forward(self, v_x):
        return self.main(v_x).view(v_x.size(0), 1)

net = Network()

i = 0
while True:
    v_in = Variable(torch.Tensor(2,1,1,1), requires_grad=True)
    grad_out = Variable(torch.ones(2,1,1,1))

    gradient = autograd.grad(outputs=net(v_in), inputs=v_in,
                             grad_outputs=grad_out,
                             create_graph=True, retain_graph=True, 
                             only_inputs=True)[0]
    gradient.mean().backward()

    i += 1
    if i % 512 == 0:
        gc.collect()
        max_mem_used = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
        print("{:.2f} MB".format(max_mem_used / 1024))
</code></pre></div>
<p dir="auto">However the leak disappears when only one convolution is used:</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="class Network(nn.Module):
    def __init__(self):
        super(Network, self).__init__()
        self.main = nn.Sequential(
            nn.Conv2d(1, 1, 1, 1),
        )
    def forward(self, v_x):
        return self.main(v_x).view(v_x.size(0), 1)"><pre class="notranslate"><code class="notranslate">class Network(nn.Module):
    def __init__(self):
        super(Network, self).__init__()
        self.main = nn.Sequential(
            nn.Conv2d(1, 1, 1, 1),
        )
    def forward(self, v_x):
        return self.main(v_x).view(v_x.size(0), 1)
</code></pre></div>
<p dir="auto">Also, if nn.Linear is used, there is no leak.</p>
<p dir="auto">Torch version: 0.4.0a0+8ebf18b</p>