<p dir="auto">For robustness of error messages, we should ensure all code paths where we throw an ErrorReport are tested and return meaningful errors.</p>
<p dir="auto">As of today, this is the coverage of the times that we <code class="notranslate">throw ErrorReport</code> for a run of <code class="notranslate">test_jit.py</code>. Format is <code class="notranslate">line, #calls, source code</code>:</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="torch/csrc/jit/script/compiler.cpp:
  148|      0|        throw ErrorReport(loc) &lt;&lt; &quot;cannot re-assign '&quot; &lt;&lt; name &lt;&lt; &quot;' to a value of type &quot; &lt;&lt; value-&gt;kind()
  153|      0|        throw ErrorReport(loc) &lt;&lt; &quot;cannot re-assign '&quot; &lt;&lt; name &lt;&lt; &quot;' because it has type &quot; &lt;&lt; value-&gt;kind()
  157|      2|        throw ErrorReport(loc) &lt;&lt; &quot;variable '&quot; &lt;&lt; name &lt;&lt; &quot;' previously has type &quot; &lt;&lt; simple_parent-&gt;type()-&gt;name()
  192|      4|      throw ErrorReport(range) &lt;&lt; &quot;undefined value &quot; &lt;&lt; ident;
  248|      0|      throw ErrorReport(loc) &lt;&lt; &quot;attributes must be scalars or lists of scalars&quot;;
  252|      0|  throw ErrorReport(loc) &lt;&lt; &quot;attributes must be constant expressions&quot;;
  497|      0|      throw ErrorReport(loc) &lt;&lt; &quot;argument 'chunks' must be a constant&quot;;
  549|      0|    throw ErrorReport(loc) &lt;&lt; &quot;unknown builtin op&quot;;
  551|      9|  throw ErrorReport(loc) &lt;&lt; &quot;arguments for call are not valid:\n&quot;
  565|      0|    throw ErrorReport(range) &lt;&lt; &quot;expected a tensor value but found a tuple&quot;;
  616|      0|        throw ErrorReport(def.params().range()) &lt;&lt; &quot;methods must have a self argument&quot;;
  713|      0|          throw ErrorReport(stmt) &lt;&lt; &quot;return statements can appear only at the end &quot;
  913|      0|      throw ErrorReport(range)
  926|      0|      throw ErrorReport(stmt)
  930|      0|      throw ErrorReport(stmt) &lt;&lt; &quot;Iteration variable unpacking is not supported&quot;;
  934|      0|      throw ErrorReport(targets[0]) &lt;&lt; &quot;Starred unpacking is currently not&quot;
  992|      0|        throw ErrorReport(assignee)
  998|      0|      throw ErrorReport(r)
 1003|      0|      throw ErrorReport(r) &lt;&lt; &quot;A Starred expression may only appear on the &quot;
 1015|      0|        throw ErrorReport(stmt)
 1042|      3|      throw ErrorReport(stmt)
 1048|      4|      throw ErrorReport(stmt)
 1060|      0|          throw ErrorReport(var) &lt;&lt; &quot;Cannot pack a tuple into a non-variable.&quot;;
 1169|      0|        throw ErrorReport(ident) &lt;&lt; &quot;print doesn't accept any keyword arguments&quot;;
 1249|      0|        throw ErrorReport(tree) &lt;&lt; &quot;Unexpected starred expansion. File a bug report.&quot;;
 1284|      0|        throw ErrorReport(tree) &lt;&lt; &quot;NYI: &quot; &lt;&lt; tree;
 1305|      0|        throw ErrorReport(input) &lt;&lt; &quot;Unrecognized type: &quot; &lt;&lt; type;
 1364|      0|      throw ErrorReport(loc)
 1461|      0|        throw ErrorReport(def) &lt;&lt; &quot;duplicate definition of function '&quot; &lt;&lt; name &lt;&lt; &quot;'&quot;;
 1495|      4|  throw ErrorReport(loc) &lt;&lt; value-&gt;type()-&gt;name() &lt;&lt; &quot; cannot be used as a tuple&quot;;
 1500|      0|    throw ErrorReport(loc) &lt;&lt; &quot;expected &quot; &lt;&lt; expected &lt;&lt; &quot; &quot; &lt;&lt; what &lt;&lt; &quot; but found &quot; &lt;&lt; actual;
torch/csrc/jit/script/compiler.h:
   41|      0|    throw ErrorReport(loc) &lt;&lt; kind() &lt;&lt; &quot; cannot be used as a value&quot;;
   46|      0|    throw ErrorReport(loc) &lt;&lt; &quot;attribute lookup is not defined on &quot; &lt;&lt; kind();
   52|      0|    throw ErrorReport(loc) &lt;&lt; kind() &lt;&lt; &quot; cannot be used as a tuple&quot;;
   78|      0|    throw ErrorReport(loc) &lt;&lt; &quot;cannot call a &quot; &lt;&lt; kind();
torch/csrc/jit/script/init.cpp:
   56|      3|      throw ErrorReport(loc) &lt;&lt; &quot;calling a Python function with an incorrect number &quot;
   61|      1|        throw ErrorReport(loc) &lt;&lt; &quot;type mismatch at argument &quot; &lt;&lt; i &lt;&lt; &quot;: expected &quot;
   70|      0|      throw ErrorReport(loc) &lt;&lt; &quot;keyword arguments in Python calls aren't supported&quot;;
  103|      0|        throw ErrorReport(loc) &lt;&lt; &quot;Python functions can currently only return Tensors&quot;;
  134|      0|      throw ErrorReport(loc) &lt;&lt; &quot;object has no attribute &quot; &lt;&lt; name;
  210|      0|  throw ErrorReport(loc) &lt;&lt; &quot;unsupported attribute lookup on &quot; &lt;&lt; py::repr(self) &lt;&lt; &quot;.&quot;;
  276|      2|        throw ErrorReport(loc) &lt;&lt; &quot;attribute '&quot; &lt;&lt; field &lt;&lt; &quot;' of type '&quot; &lt;&lt; typeString(attr) &lt;&lt; &quot;' is not usable in a script method (did you forget to add it __constants__?)&quot;;
  279|      0|    throw ErrorReport(loc) &lt;&lt; &quot;module has no attribute '&quot; &lt;&lt; field &lt;&lt; &quot;'&quot;;
torch/csrc/jit/script/module.cpp:
   34|      1|    throw ErrorReport(loc) &lt;&lt; &quot; method '&quot; &lt;&lt; callee.name()
   44|      0|    throw ErrorReport(loc) &lt;&lt; failure_messages.str();
torch/csrc/jit/script/tree_views.h:
  169|      0|      throw ErrorReport(tree) &lt;&lt; &quot;Maybe trees can have at most one subtree&quot;;
  211|      0|        throw ErrorReport(tree) &lt;&lt; kindToString(tree-&gt;kind()) &lt;&lt; &quot; is not a valid Type&quot;;
  228|      0|        throw ErrorReport(tree) &lt;&lt; kindToString(tree-&gt;kind()) &lt;&lt; &quot; is not a valid Stmt&quot;;
  266|      0|        throw ErrorReport(tree) &lt;&lt; kindToString(tree-&gt;kind()) &lt;&lt; &quot; is not a valid Expr&quot;;
  333|      0|        throw ErrorReport(tree) &lt;&lt; kindToString(tree-&gt;kind()) &lt;&lt; &quot; is not a valid ScalarType&quot;;
  451|      0|        throw ErrorReport(tree) &lt;&lt; &quot;is not a valid AssignKind&quot;;
  525|      0|          throw ErrorReport(tree) &lt;&lt; &quot;BinOp expected 2 subtrees, found &quot; &lt;&lt; tree-&gt;trees().size();
  528|      0|        throw ErrorReport(tree) &lt;&lt; kindToString(tree-&gt;kind()) &lt;&lt; &quot; is not a valid BinOp&quot;;
  548|      0|          throw ErrorReport(tree) &lt;&lt; &quot;UnaryOp expected 1 subtree, found &quot; &lt;&lt; tree-&gt;trees().size();
 551| 0| throw ErrorReport(tree) &lt;&lt; kindToString(tree-&gt;kind()) &lt;&lt; &quot; is not a valid UnaryOp&quot;;"><pre class="notranslate"><code class="notranslate">torch/csrc/jit/script/compiler.cpp:
  148|      0|        throw ErrorReport(loc) &lt;&lt; "cannot re-assign '" &lt;&lt; name &lt;&lt; "' to a value of type " &lt;&lt; value-&gt;kind()
  153|      0|        throw ErrorReport(loc) &lt;&lt; "cannot re-assign '" &lt;&lt; name &lt;&lt; "' because it has type " &lt;&lt; value-&gt;kind()
  157|      2|        throw ErrorReport(loc) &lt;&lt; "variable '" &lt;&lt; name &lt;&lt; "' previously has type " &lt;&lt; simple_parent-&gt;type()-&gt;name()
  192|      4|      throw ErrorReport(range) &lt;&lt; "undefined value " &lt;&lt; ident;
  248|      0|      throw ErrorReport(loc) &lt;&lt; "attributes must be scalars or lists of scalars";
  252|      0|  throw ErrorReport(loc) &lt;&lt; "attributes must be constant expressions";
  497|      0|      throw ErrorReport(loc) &lt;&lt; "argument 'chunks' must be a constant";
  549|      0|    throw ErrorReport(loc) &lt;&lt; "unknown builtin op";
  551|      9|  throw ErrorReport(loc) &lt;&lt; "arguments for call are not valid:\n"
  565|      0|    throw ErrorReport(range) &lt;&lt; "expected a tensor value but found a tuple";
  616|      0|        throw ErrorReport(def.params().range()) &lt;&lt; "methods must have a self argument";
  713|      0|          throw ErrorReport(stmt) &lt;&lt; "return statements can appear only at the end "
  913|      0|      throw ErrorReport(range)
  926|      0|      throw ErrorReport(stmt)
  930|      0|      throw ErrorReport(stmt) &lt;&lt; "Iteration variable unpacking is not supported";
  934|      0|      throw ErrorReport(targets[0]) &lt;&lt; "Starred unpacking is currently not"
  992|      0|        throw ErrorReport(assignee)
  998|      0|      throw ErrorReport(r)
 1003|      0|      throw ErrorReport(r) &lt;&lt; "A Starred expression may only appear on the "
 1015|      0|        throw ErrorReport(stmt)
 1042|      3|      throw ErrorReport(stmt)
 1048|      4|      throw ErrorReport(stmt)
 1060|      0|          throw ErrorReport(var) &lt;&lt; "Cannot pack a tuple into a non-variable.";
 1169|      0|        throw ErrorReport(ident) &lt;&lt; "print doesn't accept any keyword arguments";
 1249|      0|        throw ErrorReport(tree) &lt;&lt; "Unexpected starred expansion. File a bug report.";
 1284|      0|        throw ErrorReport(tree) &lt;&lt; "NYI: " &lt;&lt; tree;
 1305|      0|        throw ErrorReport(input) &lt;&lt; "Unrecognized type: " &lt;&lt; type;
 1364|      0|      throw ErrorReport(loc)
 1461|      0|        throw ErrorReport(def) &lt;&lt; "duplicate definition of function '" &lt;&lt; name &lt;&lt; "'";
 1495|      4|  throw ErrorReport(loc) &lt;&lt; value-&gt;type()-&gt;name() &lt;&lt; " cannot be used as a tuple";
 1500|      0|    throw ErrorReport(loc) &lt;&lt; "expected " &lt;&lt; expected &lt;&lt; " " &lt;&lt; what &lt;&lt; " but found " &lt;&lt; actual;
torch/csrc/jit/script/compiler.h:
   41|      0|    throw ErrorReport(loc) &lt;&lt; kind() &lt;&lt; " cannot be used as a value";
   46|      0|    throw ErrorReport(loc) &lt;&lt; "attribute lookup is not defined on " &lt;&lt; kind();
   52|      0|    throw ErrorReport(loc) &lt;&lt; kind() &lt;&lt; " cannot be used as a tuple";
   78|      0|    throw ErrorReport(loc) &lt;&lt; "cannot call a " &lt;&lt; kind();
torch/csrc/jit/script/init.cpp:
   56|      3|      throw ErrorReport(loc) &lt;&lt; "calling a Python function with an incorrect number "
   61|      1|        throw ErrorReport(loc) &lt;&lt; "type mismatch at argument " &lt;&lt; i &lt;&lt; ": expected "
   70|      0|      throw ErrorReport(loc) &lt;&lt; "keyword arguments in Python calls aren't supported";
  103|      0|        throw ErrorReport(loc) &lt;&lt; "Python functions can currently only return Tensors";
  134|      0|      throw ErrorReport(loc) &lt;&lt; "object has no attribute " &lt;&lt; name;
  210|      0|  throw ErrorReport(loc) &lt;&lt; "unsupported attribute lookup on " &lt;&lt; py::repr(self) &lt;&lt; ".";
  276|      2|        throw ErrorReport(loc) &lt;&lt; "attribute '" &lt;&lt; field &lt;&lt; "' of type '" &lt;&lt; typeString(attr) &lt;&lt; "' is not usable in a script method (did you forget to add it __constants__?)";
  279|      0|    throw ErrorReport(loc) &lt;&lt; "module has no attribute '" &lt;&lt; field &lt;&lt; "'";
torch/csrc/jit/script/module.cpp:
   34|      1|    throw ErrorReport(loc) &lt;&lt; " method '" &lt;&lt; callee.name()
   44|      0|    throw ErrorReport(loc) &lt;&lt; failure_messages.str();
torch/csrc/jit/script/tree_views.h:
  169|      0|      throw ErrorReport(tree) &lt;&lt; "Maybe trees can have at most one subtree";
  211|      0|        throw ErrorReport(tree) &lt;&lt; kindToString(tree-&gt;kind()) &lt;&lt; " is not a valid Type";
  228|      0|        throw ErrorReport(tree) &lt;&lt; kindToString(tree-&gt;kind()) &lt;&lt; " is not a valid Stmt";
  266|      0|        throw ErrorReport(tree) &lt;&lt; kindToString(tree-&gt;kind()) &lt;&lt; " is not a valid Expr";
  333|      0|        throw ErrorReport(tree) &lt;&lt; kindToString(tree-&gt;kind()) &lt;&lt; " is not a valid ScalarType";
  451|      0|        throw ErrorReport(tree) &lt;&lt; "is not a valid AssignKind";
  525|      0|          throw ErrorReport(tree) &lt;&lt; "BinOp expected 2 subtrees, found " &lt;&lt; tree-&gt;trees().size();
  528|      0|        throw ErrorReport(tree) &lt;&lt; kindToString(tree-&gt;kind()) &lt;&lt; " is not a valid BinOp";
  548|      0|          throw ErrorReport(tree) &lt;&lt; "UnaryOp expected 1 subtree, found " &lt;&lt; tree-&gt;trees().size();
 551| 0| throw ErrorReport(tree) &lt;&lt; kindToString(tree-&gt;kind()) &lt;&lt; " is not a valid UnaryOp";
</code></pre></div>
<p dir="auto">In <code class="notranslate">DataParallel</code>, gradients from GPUs are <strong>summed</strong> to update parameters. This design results in the subtlety that training convergence depends on the number of GPUs you used to train, which makes experiments less duplicable and trackable when the number of gpus is different in hardware systems or when it is controlled by augment like <code class="notranslate">--gpus 0,1</code>. Is it better to <strong>average</strong> gradients from all GPUs like <code class="notranslate">torch.nn.parallel.DistributedDataParallel</code> does, so that the training convergence is independent on the number of gpus? Thanks!</p>