title: Download websites web pages in parallel
body: <p>I have the following code and it works fine when I loop through one by one using <code>foreach</code>, but when I change it to use <code>Parallel.ForEach</code> I'm getting errors. Trying to figure out how I can correct this. FYI the <code>dParms</code> list contain unique id's.</p>
  <p>The error I'm getting is</p>
  <blockquote>
  <p>Error writing to path..... ErrorMessage:Item has already been added.
  Key in dictionary: 'IIS_WasUrlRewritten' Key being added:
  'IIS_WasUrlRewritten',</p>
  </blockquote>
  <pre><code>        private void GeneratePages(RequestStatus response, string localDir, List&lt;Parameters&gt; dParms, int total, DateTime generatedTime)
      {
          int current = 0;
          Parallel.ForEach(dParms, curJob =&gt;
          {
         
              try
              {
                  DownloadPage(localDir, curJob, generatedTime);
              }
              catch (Exception ex)
              {
                  response.Status = false;
                  response.Message = ex.Message;
                 
              }
              finally
              {
                  Interlocked.Increment(ref current);
                  if (current % 10 == 0)
                      //to do: Send Progress to UI
              }
          //});
      }
  
      private string DownloadPage(string localDir, Parameters p, DateTime generatedTime)
      {
          string strExtension = &quot;html&quot;;
          string url = string.Empty;
  
          url = this.Url.Action(&quot;MyAction&quot;, &quot;Home&quot;, new { area = &quot;&quot;, @id = p.MyId, @generatedTime = generatedTime }, this.Request.Url.Scheme);
  
          var document = new HtmlWeb().Load(url);
          
          string strFileName = url.Substring(url.LastIndexOf(&quot;/&quot;) + 1);
          strFileName = strFileName.Substring(0, strFileName.IndexOf(&quot;generatedTime&quot;) - 1);
          string strDiskFileName = strFileName.Replace(&quot;.aspx?&quot;, &quot;&quot;);
          strDiskFileName = strDiskFileName.Replace(&quot;?&quot;, &quot;&quot;);
          strDiskFileName = strDiskFileName.Replace(&quot;.aspx&quot;, &quot;&quot;);
          strDiskFileName = strDiskFileName.Replace(&quot;&amp;&quot;, &quot;&quot;);
          strDiskFileName = strDiskFileName.Replace(&quot;=&quot;, &quot;&quot;);
          strDiskFileName = strDiskFileName.Replace(&quot;%20&quot;, &quot;&quot;);
          strDiskFileName += &quot;.&quot; + strExtension;
          document.Save(localDir + strDiskFileName);
  
  
          return url;
      }
  </code></pre>