<h2 dir="auto"><g-emoji class="g-emoji" alias="bug" fallback-src="https://github.githubassets.com/images/icons/emoji/unicode/1f41b.png">🐛</g-emoji> Bug</h2>

<h2 dir="auto">To Reproduce</h2>
<p dir="auto">Steps to reproduce the behavior:</p>
<div class="highlight highlight-source-python notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="import numpy as np
import torch

a = np.array([[[  -29.8316,   506.7845,  5400.1382],
         [ -155.6416,    73.0718,  5448.8071],
         [  -52.9619,  -309.7045,  5251.0830],
         [ -300.4998,  -332.3928,  5156.6812],
         [ -258.2405,    99.6090,  5244.6816],
         [ -209.4844,   548.8338,  5290.7637],
         [ -176.7308,  -321.0486,  5203.8818],
         [ -109.1576,  -529.7281,  5123.8906],
         [ -140.1912,  -780.1214,  5074.6050],
         [ -118.9348,  -970.2283,  5058.5991],
         [   13.8925,  -279.8529,  5421.0684],
         [   35.7916,  -470.1449,  5257.7383],
         [  -19.7603,  -716.9181,  5140.2725],
         [ -259.0900,  -690.1336,  5050.5923],
         [ -370.6709,  -448.5993,  5134.1772],
         [ -462.2866,  -290.8295,  5307.6274],
         [ -153.1819,  -886.9761,  5130.1655]],

        [[  -29.6097,   507.0160,  5400.0415],
         [ -156.1013,    73.2300,  5446.2266],
         [  -52.8395,  -309.0523,  5247.8511],
         [ -299.8065,  -331.7188,  5151.9609],
         [ -258.0193,    99.6480,  5243.2402],
         [ -209.7421,   548.7822,  5290.6899],
         [ -176.3229,  -320.3856,  5199.9062],
         [ -108.0304,  -530.1070,  5123.3218],
         [ -140.6482,  -779.8441,  5071.7817],
         [ -118.2912,  -970.1854,  5056.8403],
         [   10.7354,  -281.1750,  5414.3501],
         [   40.3466,  -470.5802,  5251.2056],
         [  -19.6162,  -718.2531,  5137.8745],
         [ -258.3210,  -688.7090,  5046.1133],
         [ -368.4557,  -443.3939,  5120.0488],
         [ -460.3479,  -281.6299,  5289.6309],
         [ -153.4338,  -886.6000,  5127.5791]],

        [[  -29.7913,   507.1184,  5400.0073],
         [ -156.4464,    73.1930,  5444.3994],
         [  -52.3020,  -308.4679,  5245.2900],
         [ -299.1556,  -330.9422,  5149.0630],
         [ -258.1010,   100.1828,  5241.8062],
         [ -210.1292,   549.2088,  5290.5708],
         [ -175.7287,  -319.7050,  5197.1768],
         [ -107.6006,  -531.0508,  5125.0386],
         [ -142.8517,  -779.9221,  5071.1001],
         [ -118.7543,  -970.0274,  5056.5713],
         [   25.9413,  -271.5096,  5396.5942],
         [   45.9570,  -469.2388,  5242.0957],
         [  -21.8357,  -718.7902,  5137.6470],
         [ -258.9276,  -687.5615,  5042.6875],
         [ -361.8696,  -434.6036,  5099.1748],
         [ -458.3741,  -250.1555,  5240.7217],
         [ -154.4211,  -886.6248,  5127.2637]]])


a = torch.from_numpy(a)
a -= a[:,7:8,:]
print(a)"><pre class="notranslate"><span class="pl-k">import</span> <span class="pl-s1">numpy</span> <span class="pl-k">as</span> <span class="pl-s1">np</span>
<span class="pl-k">import</span> <span class="pl-s1">torch</span>

<span class="pl-s1">a</span> <span class="pl-c1">=</span> <span class="pl-s1">np</span>.<span class="pl-en">array</span>([[[  <span class="pl-c1">-</span><span class="pl-c1">29.8316</span>,   <span class="pl-c1">506.7845</span>,  <span class="pl-c1">5400.1382</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">155.6416</span>,    <span class="pl-c1">73.0718</span>,  <span class="pl-c1">5448.8071</span>],
         [  <span class="pl-c1">-</span><span class="pl-c1">52.9619</span>,  <span class="pl-c1">-</span><span class="pl-c1">309.7045</span>,  <span class="pl-c1">5251.0830</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">300.4998</span>,  <span class="pl-c1">-</span><span class="pl-c1">332.3928</span>,  <span class="pl-c1">5156.6812</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">258.2405</span>,    <span class="pl-c1">99.6090</span>,  <span class="pl-c1">5244.6816</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">209.4844</span>,   <span class="pl-c1">548.8338</span>,  <span class="pl-c1">5290.7637</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">176.7308</span>,  <span class="pl-c1">-</span><span class="pl-c1">321.0486</span>,  <span class="pl-c1">5203.8818</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">109.1576</span>,  <span class="pl-c1">-</span><span class="pl-c1">529.7281</span>,  <span class="pl-c1">5123.8906</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">140.1912</span>,  <span class="pl-c1">-</span><span class="pl-c1">780.1214</span>,  <span class="pl-c1">5074.6050</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">118.9348</span>,  <span class="pl-c1">-</span><span class="pl-c1">970.2283</span>,  <span class="pl-c1">5058.5991</span>],
         [   <span class="pl-c1">13.8925</span>,  <span class="pl-c1">-</span><span class="pl-c1">279.8529</span>,  <span class="pl-c1">5421.0684</span>],
         [   <span class="pl-c1">35.7916</span>,  <span class="pl-c1">-</span><span class="pl-c1">470.1449</span>,  <span class="pl-c1">5257.7383</span>],
         [  <span class="pl-c1">-</span><span class="pl-c1">19.7603</span>,  <span class="pl-c1">-</span><span class="pl-c1">716.9181</span>,  <span class="pl-c1">5140.2725</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">259.0900</span>,  <span class="pl-c1">-</span><span class="pl-c1">690.1336</span>,  <span class="pl-c1">5050.5923</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">370.6709</span>,  <span class="pl-c1">-</span><span class="pl-c1">448.5993</span>,  <span class="pl-c1">5134.1772</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">462.2866</span>,  <span class="pl-c1">-</span><span class="pl-c1">290.8295</span>,  <span class="pl-c1">5307.6274</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">153.1819</span>,  <span class="pl-c1">-</span><span class="pl-c1">886.9761</span>,  <span class="pl-c1">5130.1655</span>]],

        [[  <span class="pl-c1">-</span><span class="pl-c1">29.6097</span>,   <span class="pl-c1">507.0160</span>,  <span class="pl-c1">5400.0415</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">156.1013</span>,    <span class="pl-c1">73.2300</span>,  <span class="pl-c1">5446.2266</span>],
         [  <span class="pl-c1">-</span><span class="pl-c1">52.8395</span>,  <span class="pl-c1">-</span><span class="pl-c1">309.0523</span>,  <span class="pl-c1">5247.8511</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">299.8065</span>,  <span class="pl-c1">-</span><span class="pl-c1">331.7188</span>,  <span class="pl-c1">5151.9609</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">258.0193</span>,    <span class="pl-c1">99.6480</span>,  <span class="pl-c1">5243.2402</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">209.7421</span>,   <span class="pl-c1">548.7822</span>,  <span class="pl-c1">5290.6899</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">176.3229</span>,  <span class="pl-c1">-</span><span class="pl-c1">320.3856</span>,  <span class="pl-c1">5199.9062</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">108.0304</span>,  <span class="pl-c1">-</span><span class="pl-c1">530.1070</span>,  <span class="pl-c1">5123.3218</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">140.6482</span>,  <span class="pl-c1">-</span><span class="pl-c1">779.8441</span>,  <span class="pl-c1">5071.7817</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">118.2912</span>,  <span class="pl-c1">-</span><span class="pl-c1">970.1854</span>,  <span class="pl-c1">5056.8403</span>],
         [   <span class="pl-c1">10.7354</span>,  <span class="pl-c1">-</span><span class="pl-c1">281.1750</span>,  <span class="pl-c1">5414.3501</span>],
         [   <span class="pl-c1">40.3466</span>,  <span class="pl-c1">-</span><span class="pl-c1">470.5802</span>,  <span class="pl-c1">5251.2056</span>],
         [  <span class="pl-c1">-</span><span class="pl-c1">19.6162</span>,  <span class="pl-c1">-</span><span class="pl-c1">718.2531</span>,  <span class="pl-c1">5137.8745</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">258.3210</span>,  <span class="pl-c1">-</span><span class="pl-c1">688.7090</span>,  <span class="pl-c1">5046.1133</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">368.4557</span>,  <span class="pl-c1">-</span><span class="pl-c1">443.3939</span>,  <span class="pl-c1">5120.0488</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">460.3479</span>,  <span class="pl-c1">-</span><span class="pl-c1">281.6299</span>,  <span class="pl-c1">5289.6309</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">153.4338</span>,  <span class="pl-c1">-</span><span class="pl-c1">886.6000</span>,  <span class="pl-c1">5127.5791</span>]],

        [[  <span class="pl-c1">-</span><span class="pl-c1">29.7913</span>,   <span class="pl-c1">507.1184</span>,  <span class="pl-c1">5400.0073</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">156.4464</span>,    <span class="pl-c1">73.1930</span>,  <span class="pl-c1">5444.3994</span>],
         [  <span class="pl-c1">-</span><span class="pl-c1">52.3020</span>,  <span class="pl-c1">-</span><span class="pl-c1">308.4679</span>,  <span class="pl-c1">5245.2900</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">299.1556</span>,  <span class="pl-c1">-</span><span class="pl-c1">330.9422</span>,  <span class="pl-c1">5149.0630</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">258.1010</span>,   <span class="pl-c1">100.1828</span>,  <span class="pl-c1">5241.8062</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">210.1292</span>,   <span class="pl-c1">549.2088</span>,  <span class="pl-c1">5290.5708</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">175.7287</span>,  <span class="pl-c1">-</span><span class="pl-c1">319.7050</span>,  <span class="pl-c1">5197.1768</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">107.6006</span>,  <span class="pl-c1">-</span><span class="pl-c1">531.0508</span>,  <span class="pl-c1">5125.0386</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">142.8517</span>,  <span class="pl-c1">-</span><span class="pl-c1">779.9221</span>,  <span class="pl-c1">5071.1001</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">118.7543</span>,  <span class="pl-c1">-</span><span class="pl-c1">970.0274</span>,  <span class="pl-c1">5056.5713</span>],
         [   <span class="pl-c1">25.9413</span>,  <span class="pl-c1">-</span><span class="pl-c1">271.5096</span>,  <span class="pl-c1">5396.5942</span>],
         [   <span class="pl-c1">45.9570</span>,  <span class="pl-c1">-</span><span class="pl-c1">469.2388</span>,  <span class="pl-c1">5242.0957</span>],
         [  <span class="pl-c1">-</span><span class="pl-c1">21.8357</span>,  <span class="pl-c1">-</span><span class="pl-c1">718.7902</span>,  <span class="pl-c1">5137.6470</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">258.9276</span>,  <span class="pl-c1">-</span><span class="pl-c1">687.5615</span>,  <span class="pl-c1">5042.6875</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">361.8696</span>,  <span class="pl-c1">-</span><span class="pl-c1">434.6036</span>,  <span class="pl-c1">5099.1748</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">458.3741</span>,  <span class="pl-c1">-</span><span class="pl-c1">250.1555</span>,  <span class="pl-c1">5240.7217</span>],
         [ <span class="pl-c1">-</span><span class="pl-c1">154.4211</span>,  <span class="pl-c1">-</span><span class="pl-c1">886.6248</span>,  <span class="pl-c1">5127.2637</span>]]])


<span class="pl-s1">a</span> <span class="pl-c1">=</span> <span class="pl-s1">torch</span>.<span class="pl-en">from_numpy</span>(<span class="pl-s1">a</span>)
<span class="pl-s1">a</span> <span class="pl-c1">-=</span> <span class="pl-s1">a</span>[:,<span class="pl-c1">7</span>:<span class="pl-c1">8</span>,:]
<span class="pl-en">print</span>(<span class="pl-s1">a</span>)</pre></div>
<p dir="auto">output</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="tensor([[[   79.3260,  1036.5126,   276.2476],
         [  -46.4840,   602.7999,   324.9165],
         [   56.1957,   220.0236,   127.1924],
         [ -191.3422,   197.3353,    32.7906],
         [ -149.0829,   629.3371,   120.7910],
         [ -100.3268,  1078.5619,   166.8731],
         [  -67.5732,   208.6795,    79.9912],
         [    0.0000,     0.0000,     0.0000],
         [ -140.1912,  -780.1214,  5074.6050], #&lt;---here
         [ -118.9348,  -970.2283,  5058.5991],
         [   13.8925,  -279.8529,  5421.0684],
         [   35.7916,  -470.1449,  5257.7383],
         [  -19.7603,  -716.9181,  5140.2725],
         [ -259.0900,  -690.1336,  5050.5923],
         [ -370.6709,  -448.5993,  5134.1772],
         [ -462.2866,  -290.8295,  5307.6274],
         [ -153.1819,  -886.9761,  5130.1655]],

        [[   78.4207,  1037.1230,   276.7197],
         [  -48.0709,   603.3370,   322.9048],
         [   55.1909,   221.0547,   124.5293],
         [ -191.7761,   198.3882,    28.6391],
         [ -149.9889,   629.7550,   119.9184],
         [ -101.7117,  1078.8892,   167.3681],
         [  -68.2925,   209.7214,    76.5844],
         [    0.0000,     0.0000,     0.0000],
         [ -140.6482,  -779.8441,  5071.7817],
         [ -118.2912,  -970.1854,  5056.8403],
         [   10.7354,  -281.1750,  5414.3501],
         [   40.3466,  -470.5802,  5251.2056],
         [  -19.6162,  -718.2531,  5137.8745],
         [ -258.3210,  -688.7090,  5046.1133],
         [ -368.4557,  -443.3939,  5120.0488],
         [ -460.3479,  -281.6299,  5289.6309],
         [ -153.4338,  -886.6000,  5127.5791]],

        [[   77.8093,  1038.1692,   274.9687],
         [  -48.8458,   604.2438,   319.3608],
         [   55.2986,   222.5829,   120.2514],
         [ -191.5550,   200.1086,    24.0244],
         [ -150.5004,   631.2336,   116.7676],
         [ -102.5286,  1080.2596,   165.5322],
         [  -68.1281,   211.3458,    72.1382],
         [    0.0000,     0.0000,     0.0000],
         [ -142.8517,  -779.9221,  5071.1001],
         [ -118.7543,  -970.0274,  5056.5713],
         [   25.9413,  -271.5096,  5396.5942],
         [   45.9570,  -469.2388,  5242.0957],
         [  -21.8357,  -718.7902,  5137.6470],
         [ -258.9276,  -687.5615,  5042.6875],
         [ -361.8696,  -434.6036,  5099.1748],
         [ -458.3741,  -250.1555,  5240.7217],
         [ -154.4211,  -886.6248,  5127.2637]]], dtype=torch.float64)"><pre class="notranslate"><code class="notranslate">tensor([[[   79.3260,  1036.5126,   276.2476],
         [  -46.4840,   602.7999,   324.9165],
         [   56.1957,   220.0236,   127.1924],
         [ -191.3422,   197.3353,    32.7906],
         [ -149.0829,   629.3371,   120.7910],
         [ -100.3268,  1078.5619,   166.8731],
         [  -67.5732,   208.6795,    79.9912],
         [    0.0000,     0.0000,     0.0000],
         [ -140.1912,  -780.1214,  5074.6050], #&lt;---here
         [ -118.9348,  -970.2283,  5058.5991],
         [   13.8925,  -279.8529,  5421.0684],
         [   35.7916,  -470.1449,  5257.7383],
         [  -19.7603,  -716.9181,  5140.2725],
         [ -259.0900,  -690.1336,  5050.5923],
         [ -370.6709,  -448.5993,  5134.1772],
         [ -462.2866,  -290.8295,  5307.6274],
         [ -153.1819,  -886.9761,  5130.1655]],

        [[   78.4207,  1037.1230,   276.7197],
         [  -48.0709,   603.3370,   322.9048],
         [   55.1909,   221.0547,   124.5293],
         [ -191.7761,   198.3882,    28.6391],
         [ -149.9889,   629.7550,   119.9184],
         [ -101.7117,  1078.8892,   167.3681],
         [  -68.2925,   209.7214,    76.5844],
         [    0.0000,     0.0000,     0.0000],
         [ -140.6482,  -779.8441,  5071.7817],
         [ -118.2912,  -970.1854,  5056.8403],
         [   10.7354,  -281.1750,  5414.3501],
         [   40.3466,  -470.5802,  5251.2056],
         [  -19.6162,  -718.2531,  5137.8745],
         [ -258.3210,  -688.7090,  5046.1133],
         [ -368.4557,  -443.3939,  5120.0488],
         [ -460.3479,  -281.6299,  5289.6309],
         [ -153.4338,  -886.6000,  5127.5791]],

        [[   77.8093,  1038.1692,   274.9687],
         [  -48.8458,   604.2438,   319.3608],
         [   55.2986,   222.5829,   120.2514],
         [ -191.5550,   200.1086,    24.0244],
         [ -150.5004,   631.2336,   116.7676],
         [ -102.5286,  1080.2596,   165.5322],
         [  -68.1281,   211.3458,    72.1382],
         [    0.0000,     0.0000,     0.0000],
         [ -142.8517,  -779.9221,  5071.1001],
         [ -118.7543,  -970.0274,  5056.5713],
         [   25.9413,  -271.5096,  5396.5942],
         [   45.9570,  -469.2388,  5242.0957],
         [  -21.8357,  -718.7902,  5137.6470],
         [ -258.9276,  -687.5615,  5042.6875],
         [ -361.8696,  -434.6036,  5099.1748],
         [ -458.3741,  -250.1555,  5240.7217],
         [ -154.4211,  -886.6248,  5127.2637]]], dtype=torch.float64)
</code></pre></div>

<h2 dir="auto">Expected behavior</h2>
<div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="tensor([[[   79.3260,  1036.5126,   276.2476],
         [  -46.4840,   602.7999,   324.9165],
         [   56.1957,   220.0236,   127.1924],
         [ -191.3422,   197.3353,    32.7906],
         [ -149.0829,   629.3371,   120.7910],
         [ -100.3268,  1078.5619,   166.8731],
         [  -67.5732,   208.6795,    79.9912],
         [    0.0000,     0.0000,     0.0000],
         [ -140.1912,  -780.1214,  5074.6050], #&lt;---here
         [ -118.9348,  -970.2283,  5058.5991],
         [   13.8925,  -279.8529,  5421.0684],
         [   35.7916,  -470.1449,  5257.7383],
         [  -19.7603,  -716.9181,  5140.2725],
         [ -259.0900,  -690.1336,  5050.5923],
         [ -370.6709,  -448.5993,  5134.1772],
         [ -462.2866,  -290.8295,  5307.6274],
         [ -153.1819,  -886.9761,  5130.1655]],
..."><pre class="notranslate">tensor([[[   79.3260,  1036.5126,   276.2476],
         [  -46.4840,   602.7999,   324.9165],
         [   56.1957,   220.0236,   127.1924],
         [ -191.3422,   197.3353,    32.7906],
         [ -149.0829,   629.3371,   120.7910],
         [ -100.3268,  1078.5619,   166.8731],
         [  -67.5732,   208.6795,    79.9912],
         [    0.0000,     0.0000,     0.0000],
         [ -140.1912,  -780.1214,  5074.6050], <span class="pl-c"><span class="pl-c">#</span>&lt;---here</span>
         [ -118.9348,  -970.2283,  5058.5991],
         [   13.8925,  -279.8529,  5421.0684],
         [   35.7916,  -470.1449,  5257.7383],
         [  -19.7603,  -716.9181,  5140.2725],
         [ -259.0900,  -690.1336,  5050.5923],
         [ -370.6709,  -448.5993,  5134.1772],
         [ -462.2866,  -290.8295,  5307.6274],
         [ -153.1819,  -886.9761,  5130.1655]],
...</pre></div>

<h2 dir="auto">Environment</h2>
<div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="PyTorch version: 0.4.0
Is debug build: No
CUDA used to build PyTorch: 8.0.61

OS: Ubuntu 16.04.5 LTS
GCC version: (Ubuntu 5.4.0-6ubuntu1~16.04.10) 5.4.0 20160609
CMake version: version 3.5.1

Python version: 3.6
Is CUDA available: Yes
CUDA runtime version: 9.0.176
GPU models and configuration:
GPU 0: TITAN Xp
GPU 1: TITAN Xp
GPU 2: TITAN Xp
GPU 3: TITAN Xp
GPU 4: TITAN Xp
GPU 5: TITAN Xp
GPU 6: TITAN Xp
GPU 7: TITAN Xp

Nvidia driver version: 396.54
cuDNN version: Probably one of the following:
/usr/local/cuda-9.0/lib64/libcudnn.so.7.3.0
/usr/local/cuda-9.0/lib64/libcudnn_static.a
/usr/local/cuda-9.2/lib64/libcudnn.so.7.2.1
/usr/local/cuda-9.2/lib64/libcudnn_static.a

Versions of relevant libraries:
[pip] numpy (1.15.2)
[pip] torch (0.4.0)
[pip] torchfile (0.1.0)
[pip] torchsample (0.1.3, /home/nonews/src/torchsample)
[pip] torchvision (0.2.1)
[conda] torch                     0.4.0                     &lt;pip&gt;
[conda] torchfile                 0.1.0                     &lt;pip&gt;
[conda] torchsample               0.1.3                     &lt;pip&gt;
[conda] torchvision               0.2.1                     &lt;pip&gt;"><pre class="notranslate">PyTorch version: 0.4.0
Is debug build: No
CUDA used to build PyTorch: 8.0.61

OS: Ubuntu 16.04.5 LTS
GCC version: (Ubuntu 5.4.0-6ubuntu1~16.04.10) 5.4.0 20160609
CMake version: version 3.5.1

Python version: 3.6
Is CUDA available: Yes
CUDA runtime version: 9.0.176
GPU models and configuration:
GPU 0: TITAN Xp
GPU 1: TITAN Xp
GPU 2: TITAN Xp
GPU 3: TITAN Xp
GPU 4: TITAN Xp
GPU 5: TITAN Xp
GPU 6: TITAN Xp
GPU 7: TITAN Xp

Nvidia driver version: 396.54
cuDNN version: Probably one of the following:
/usr/local/cuda-9.0/lib64/libcudnn.so.7.3.0
/usr/local/cuda-9.0/lib64/libcudnn_static.a
/usr/local/cuda-9.2/lib64/libcudnn.so.7.2.1
/usr/local/cuda-9.2/lib64/libcudnn_static.a

Versions of relevant libraries:
[pip] numpy (1.15.2)
[pip] torch (0.4.0)
[pip] torchfile (0.1.0)
[pip] torchsample (0.1.3, /home/nonews/src/torchsample)
[pip] torchvision (0.2.1)
[conda] torch                     0.4.0                     <span class="pl-k">&lt;</span>pip<span class="pl-k">&gt;</span>
[conda] torchfile                 0.1.0                     <span class="pl-k">&lt;</span>pip<span class="pl-k">&gt;</span>
[conda] torchsample               0.1.3                     <span class="pl-k">&lt;</span>pip<span class="pl-k">&gt;</span>
[conda] torchvision               0.2.1                     <span class="pl-k">&lt;</span>pip<span class="pl-k">&gt;</span></pre></div>
<h2 dir="auto">Additional context</h2>
<p dir="auto">Ptrblck was able to reproduce this issue in 1.0.0a0+dfad8b6 and 1.0.0.dev20181007 on Nov 11.</p>

<p dir="auto">The output is caused by changing of tensor <code class="notranslate">a</code> before the op finish.</p>
<h2 dir="auto">Issue description</h2>
<p dir="auto">I have a 80x256x256x10 FC network I'm using for policy gradient, but when I do a forward pass on it it takes 70-100ms (!!) to execute. Puzzling-ly, after some condition that I can't identify, the forward pass suddenly speeds up permanently to &lt; 1-3ms (I couldn't get a reproducible example of that though). Exiting the code during execution seems to imply that it's spending a lot of time in <code class="notranslate">torch.addmm(bias, input, weight.t())</code>. Adding <code class="notranslate">pytorch.set_num_threads(1)</code> seems to fix it, but I don't know why. Sorry if this is a duplicate (I suspect it may be) but I looked and couldn't find anything in the issues</p>
<h2 dir="auto">Code example</h2>
<p dir="auto">Here's a (sort of) minimal example that reproduces the slowness:</p>
<div class="highlight highlight-source-python notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import time

class GaussianMLP(nn.Module):
    def __init__(self, num_inputs, action_dim, init_std=1, learn_std=True, nonlinearity=&quot;tanh&quot;):
        super(GaussianMLP, self).__init__()

        actor_dims = (256, 256)
        critic_dims = (256, 256)

        # create actor network
        self.actor_layers = nn.ModuleList()
        self.actor_layers += [nn.Linear(num_inputs, actor_dims[0])]
        for l in range(len(actor_dims) - 1):
            in_dim = actor_dims[l]
            out_dim = actor_dims[l + 1]
            self.actor_layers += [nn.Linear(in_dim, out_dim)]
        
        self.mean = nn.Linear(actor_dims[-1], action_dim)

        # create critic network
        self.critic_layers = nn.ModuleList()
        self.critic_layers += [nn.Linear(num_inputs, critic_dims[0])]
        for l in range(len(critic_dims) - 1):
            in_dim = critic_dims[l]
            out_dim = critic_dims[l + 1]
            self.critic_layers += [nn.Linear(in_dim, out_dim)]

        self.vf = nn.Linear(critic_dims[-1], 1)

        if nonlinearity == &quot;relu&quot;:
            self.nonlinearity = F.relu
        else:
            self.nonlinearity = torch.tanh
        
        self.train()

    def forward(self, inputs):
        start = time.time()
        x = inputs
        for l in self.critic_layers:
            x = self.nonlinearity(l(x))
        value = self.vf(x)

        x = inputs
        for l in self.actor_layers:
            x = self.nonlinearity(l(x))
        x = self.mean(x)

        x = torch.tanh(x)
        print(time.time() - start)

        return value, x

policy = GaussianMLP(80, 10, nonlinearity=&quot;relu&quot;, init_std=np.exp(-2), learn_std=False)

for _ in range(100):
    s = torch.rand(1, 80)
    policy(s)"><pre class="notranslate"><span class="pl-k">import</span> <span class="pl-s1">torch</span>
<span class="pl-k">import</span> <span class="pl-s1">torch</span>.<span class="pl-s1">nn</span> <span class="pl-k">as</span> <span class="pl-s1">nn</span>
<span class="pl-k">import</span> <span class="pl-s1">torch</span>.<span class="pl-s1">nn</span>.<span class="pl-s1">functional</span> <span class="pl-k">as</span> <span class="pl-v">F</span>
<span class="pl-k">import</span> <span class="pl-s1">numpy</span> <span class="pl-k">as</span> <span class="pl-s1">np</span>
<span class="pl-k">import</span> <span class="pl-s1">time</span>

<span class="pl-k">class</span> <span class="pl-v">GaussianMLP</span>(<span class="pl-s1">nn</span>.<span class="pl-v">Module</span>):
    <span class="pl-k">def</span> <span class="pl-en">__init__</span>(<span class="pl-s1">self</span>, <span class="pl-s1">num_inputs</span>, <span class="pl-s1">action_dim</span>, <span class="pl-s1">init_std</span><span class="pl-c1">=</span><span class="pl-c1">1</span>, <span class="pl-s1">learn_std</span><span class="pl-c1">=</span><span class="pl-c1">True</span>, <span class="pl-s1">nonlinearity</span><span class="pl-c1">=</span><span class="pl-s">"tanh"</span>):
        <span class="pl-en">super</span>(<span class="pl-v">GaussianMLP</span>, <span class="pl-s1">self</span>).<span class="pl-en">__init__</span>()

        <span class="pl-s1">actor_dims</span> <span class="pl-c1">=</span> (<span class="pl-c1">256</span>, <span class="pl-c1">256</span>)
        <span class="pl-s1">critic_dims</span> <span class="pl-c1">=</span> (<span class="pl-c1">256</span>, <span class="pl-c1">256</span>)

        <span class="pl-c"># create actor network</span>
        <span class="pl-s1">self</span>.<span class="pl-s1">actor_layers</span> <span class="pl-c1">=</span> <span class="pl-s1">nn</span>.<span class="pl-v">ModuleList</span>()
        <span class="pl-s1">self</span>.<span class="pl-s1">actor_layers</span> <span class="pl-c1">+=</span> [<span class="pl-s1">nn</span>.<span class="pl-v">Linear</span>(<span class="pl-s1">num_inputs</span>, <span class="pl-s1">actor_dims</span>[<span class="pl-c1">0</span>])]
        <span class="pl-k">for</span> <span class="pl-s1">l</span> <span class="pl-c1">in</span> <span class="pl-en">range</span>(<span class="pl-en">len</span>(<span class="pl-s1">actor_dims</span>) <span class="pl-c1">-</span> <span class="pl-c1">1</span>):
            <span class="pl-s1">in_dim</span> <span class="pl-c1">=</span> <span class="pl-s1">actor_dims</span>[<span class="pl-s1">l</span>]
            <span class="pl-s1">out_dim</span> <span class="pl-c1">=</span> <span class="pl-s1">actor_dims</span>[<span class="pl-s1">l</span> <span class="pl-c1">+</span> <span class="pl-c1">1</span>]
            <span class="pl-s1">self</span>.<span class="pl-s1">actor_layers</span> <span class="pl-c1">+=</span> [<span class="pl-s1">nn</span>.<span class="pl-v">Linear</span>(<span class="pl-s1">in_dim</span>, <span class="pl-s1">out_dim</span>)]
        
        <span class="pl-s1">self</span>.<span class="pl-s1">mean</span> <span class="pl-c1">=</span> <span class="pl-s1">nn</span>.<span class="pl-v">Linear</span>(<span class="pl-s1">actor_dims</span>[<span class="pl-c1">-</span><span class="pl-c1">1</span>], <span class="pl-s1">action_dim</span>)

        <span class="pl-c"># create critic network</span>
        <span class="pl-s1">self</span>.<span class="pl-s1">critic_layers</span> <span class="pl-c1">=</span> <span class="pl-s1">nn</span>.<span class="pl-v">ModuleList</span>()
        <span class="pl-s1">self</span>.<span class="pl-s1">critic_layers</span> <span class="pl-c1">+=</span> [<span class="pl-s1">nn</span>.<span class="pl-v">Linear</span>(<span class="pl-s1">num_inputs</span>, <span class="pl-s1">critic_dims</span>[<span class="pl-c1">0</span>])]
        <span class="pl-k">for</span> <span class="pl-s1">l</span> <span class="pl-c1">in</span> <span class="pl-en">range</span>(<span class="pl-en">len</span>(<span class="pl-s1">critic_dims</span>) <span class="pl-c1">-</span> <span class="pl-c1">1</span>):
            <span class="pl-s1">in_dim</span> <span class="pl-c1">=</span> <span class="pl-s1">critic_dims</span>[<span class="pl-s1">l</span>]
            <span class="pl-s1">out_dim</span> <span class="pl-c1">=</span> <span class="pl-s1">critic_dims</span>[<span class="pl-s1">l</span> <span class="pl-c1">+</span> <span class="pl-c1">1</span>]
            <span class="pl-s1">self</span>.<span class="pl-s1">critic_layers</span> <span class="pl-c1">+=</span> [<span class="pl-s1">nn</span>.<span class="pl-v">Linear</span>(<span class="pl-s1">in_dim</span>, <span class="pl-s1">out_dim</span>)]

        <span class="pl-s1">self</span>.<span class="pl-s1">vf</span> <span class="pl-c1">=</span> <span class="pl-s1">nn</span>.<span class="pl-v">Linear</span>(<span class="pl-s1">critic_dims</span>[<span class="pl-c1">-</span><span class="pl-c1">1</span>], <span class="pl-c1">1</span>)

        <span class="pl-k">if</span> <span class="pl-s1">nonlinearity</span> <span class="pl-c1">==</span> <span class="pl-s">"relu"</span>:
            <span class="pl-s1">self</span>.<span class="pl-s1">nonlinearity</span> <span class="pl-c1">=</span> <span class="pl-v">F</span>.<span class="pl-s1">relu</span>
        <span class="pl-k">else</span>:
            <span class="pl-s1">self</span>.<span class="pl-s1">nonlinearity</span> <span class="pl-c1">=</span> <span class="pl-s1">torch</span>.<span class="pl-s1">tanh</span>
        
        <span class="pl-s1">self</span>.<span class="pl-en">train</span>()

    <span class="pl-k">def</span> <span class="pl-en">forward</span>(<span class="pl-s1">self</span>, <span class="pl-s1">inputs</span>):
        <span class="pl-s1">start</span> <span class="pl-c1">=</span> <span class="pl-s1">time</span>.<span class="pl-en">time</span>()
        <span class="pl-s1">x</span> <span class="pl-c1">=</span> <span class="pl-s1">inputs</span>
        <span class="pl-k">for</span> <span class="pl-s1">l</span> <span class="pl-c1">in</span> <span class="pl-s1">self</span>.<span class="pl-s1">critic_layers</span>:
            <span class="pl-s1">x</span> <span class="pl-c1">=</span> <span class="pl-s1">self</span>.<span class="pl-en">nonlinearity</span>(<span class="pl-en">l</span>(<span class="pl-s1">x</span>))
        <span class="pl-s1">value</span> <span class="pl-c1">=</span> <span class="pl-s1">self</span>.<span class="pl-en">vf</span>(<span class="pl-s1">x</span>)

        <span class="pl-s1">x</span> <span class="pl-c1">=</span> <span class="pl-s1">inputs</span>
        <span class="pl-k">for</span> <span class="pl-s1">l</span> <span class="pl-c1">in</span> <span class="pl-s1">self</span>.<span class="pl-s1">actor_layers</span>:
            <span class="pl-s1">x</span> <span class="pl-c1">=</span> <span class="pl-s1">self</span>.<span class="pl-en">nonlinearity</span>(<span class="pl-en">l</span>(<span class="pl-s1">x</span>))
        <span class="pl-s1">x</span> <span class="pl-c1">=</span> <span class="pl-s1">self</span>.<span class="pl-en">mean</span>(<span class="pl-s1">x</span>)

        <span class="pl-s1">x</span> <span class="pl-c1">=</span> <span class="pl-s1">torch</span>.<span class="pl-en">tanh</span>(<span class="pl-s1">x</span>)
        <span class="pl-en">print</span>(<span class="pl-s1">time</span>.<span class="pl-en">time</span>() <span class="pl-c1">-</span> <span class="pl-s1">start</span>)

        <span class="pl-k">return</span> <span class="pl-s1">value</span>, <span class="pl-s1">x</span>

<span class="pl-s1">policy</span> <span class="pl-c1">=</span> <span class="pl-v">GaussianMLP</span>(<span class="pl-c1">80</span>, <span class="pl-c1">10</span>, <span class="pl-s1">nonlinearity</span><span class="pl-c1">=</span><span class="pl-s">"relu"</span>, <span class="pl-s1">init_std</span><span class="pl-c1">=</span><span class="pl-s1">np</span>.<span class="pl-en">exp</span>(<span class="pl-c1">-</span><span class="pl-c1">2</span>), <span class="pl-s1">learn_std</span><span class="pl-c1">=</span><span class="pl-c1">False</span>)

<span class="pl-k">for</span> <span class="pl-s1">_</span> <span class="pl-c1">in</span> <span class="pl-en">range</span>(<span class="pl-c1">100</span>):
    <span class="pl-s1">s</span> <span class="pl-c1">=</span> <span class="pl-s1">torch</span>.<span class="pl-en">rand</span>(<span class="pl-c1">1</span>, <span class="pl-c1">80</span>)
    <span class="pl-en">policy</span>(<span class="pl-s1">s</span>)</pre></div>
<p dir="auto">And here's an iPython stacktrace from randomly exiting that code while it's running:</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="    61 for _ in range(100):
     62     s = torch.rand(1, 80)
---&gt; 63     policy(s)

~/.local/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    475             result = self._slow_forward(*input, **kwargs)
    476         else:
--&gt; 477             result = self.forward(*input, **kwargs)
    478         for hook in self._forward_hooks.values():
    479             hook_result = hook(self, input, result)

~/p-morais/cassie-sim-to-real/gaussian_mlp.py in forward(self, inputs)
     49         x = inputs
     50         for l in self.actor_layers:
---&gt; 51             x = self.nonlinearity(l(x))
     52         x = self.mean(x)
     53 

~/.local/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    475             result = self._slow_forward(*input, **kwargs)
    476         else:
--&gt; 477             result = self.forward(*input, **kwargs)
    478         for hook in self._forward_hooks.values():
    479             hook_result = hook(self, input, result)

~/.local/lib/python3.6/site-packages/torch/nn/modules/linear.py in forward(self, input)
     53 
     54     def forward(self, input):
---&gt; 55         return F.linear(input, self.weight, self.bias)
     56 
     57     def extra_repr(self):

~/.local/lib/python3.6/site-packages/torch/nn/functional.py in linear(input, weight, bias)
   1022     if input.dim() == 2 and bias is not None:
   1023         # fused op is marginally faster
-&gt; 1024         return torch.addmm(bias, input, weight.t())
   1025 
   1026     output = input.matmul(weight.t()))"><pre lang="iPython" class="notranslate"><code class="notranslate">    61 for _ in range(100):
     62     s = torch.rand(1, 80)
---&gt; 63     policy(s)

~/.local/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    475             result = self._slow_forward(*input, **kwargs)
    476         else:
--&gt; 477             result = self.forward(*input, **kwargs)
    478         for hook in self._forward_hooks.values():
    479             hook_result = hook(self, input, result)

~/p-morais/cassie-sim-to-real/gaussian_mlp.py in forward(self, inputs)
     49         x = inputs
     50         for l in self.actor_layers:
---&gt; 51             x = self.nonlinearity(l(x))
     52         x = self.mean(x)
     53 

~/.local/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    475             result = self._slow_forward(*input, **kwargs)
    476         else:
--&gt; 477             result = self.forward(*input, **kwargs)
    478         for hook in self._forward_hooks.values():
    479             hook_result = hook(self, input, result)

~/.local/lib/python3.6/site-packages/torch/nn/modules/linear.py in forward(self, input)
     53 
     54     def forward(self, input):
---&gt; 55         return F.linear(input, self.weight, self.bias)
     56 
     57     def extra_repr(self):

~/.local/lib/python3.6/site-packages/torch/nn/functional.py in linear(input, weight, bias)
   1022     if input.dim() == 2 and bias is not None:
   1023         # fused op is marginally faster
-&gt; 1024         return torch.addmm(bias, input, weight.t())
   1025 
   1026     output = input.matmul(weight.t()))
</code></pre></div>
<h2 dir="auto">System Info</h2>
<p dir="auto">PyTorch version: 0.4.1<br>
Is debug build: No<br>
CUDA used to build PyTorch: 9.0.176</p>
<p dir="auto">OS: Ubuntu 18.04.1 LTS<br>
GCC version: (Ubuntu 7.3.0-27ubuntu1~18.04) 7.3.0<br>
CMake version: version 3.10.2</p>
<p dir="auto">Python version: 3.6<br>
Is CUDA available: Yes<br>
CUDA runtime version: Could not collect<br>
GPU models and configuration: GPU 0: TITAN X (Pascal)<br>
Nvidia driver version: 390.77<br>
cuDNN version: Could not collect</p>
<p dir="auto">Versions of relevant libraries:<br>
[pip3] numpy (1.15.3)<br>
[pip3] torch (1.0.0a0+c029c83)<br>
[pip3] torchvision (0.2.1)<br>
[conda] pytorch                   0.4.0           py36_cuda8.0.61_cudnn7.1.2_1    pytorch<br>
[conda] pytorch-cpu               0.4.1                py36_cpu_1    pytorch<br>
[conda] torchfile                 0.1.0                     <br>
[conda] torchvision-cpu           0.2.1                    py36_1    pytorch</p>