title: Memory leak, using HttpClient and limited tasks
body: <p>I'm trying to run 10 concurrent web requests at a time, maximum. I'm controlling the number of web requests by a semaphore (see code below). However, when running, the memory grows - and eventually it has grown so much, that the application becomes real slow.</p>
  
  <p>Firing up  <code>.NET Memory Profiler</code> shows the following 5 entries, which seems to be the cause (and also grows through out the applications lifetime):</p>
  
  <p><img src="https://i.stack.imgur.com/RcSyl.png" alt="enter image description here"></p>
  
  <p>At first glance, I just thought that the GC wasn't kicking in. But adding the following code and still see no effect worried me a little:</p>
  
  <pre><code>while (true)
  {
      Console.ReadKey();
  
      GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced);
      GC.WaitForPendingFinalizers();
      GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced);
  }
  </code></pre>
  
  <p>(taken from @Jon Skeets answer in <a href="https://stackoverflow.com/questions/653730/gc-collect-in-a-loop">GC.Collect in a loop?</a>)</p>
  
  <hr>
  
  <p>The full code:</p>
  
  <pre><code>static async Task&lt;Page&gt; GetPageAsync(this HttpClient client, long maxContentLength, Uri requestUri)
  {
      var loadTimeMeasure = Stopwatch.StartNew();
  
      var response = await client.GetAsync(requestUri).ConfigureAwait(false);
  
      if (response.IsSuccessStatusCode)
      {
          using (response.Content)
          {
              var location = response.Headers.Location;
  
              var responseUri =
                  location == null ? requestUri :
                  location.IsAbsoluteUri ? location :
                  new Uri(requestUri, location);
  
              var contentType = response.Content.Headers.ContentType.MediaType;
  
              var contentLength = response.Content.Headers.ContentLength.HasValue
                  ? response.Content.Headers.ContentLength.Value : 0;
  
              Func&lt;HtmlNode, long, Page&gt; BuildPage = (htmlNode, loadTime) =&gt; new Page
              {
                  RequestUri = requestUri,
                  ResponseUri = responseUri,
                  HTML = htmlNode,
                  LoadTime = loadTime,
                  StatusCode = response.StatusCode,
                  Headers = ExtractHeaders(response),
                  ContentLength = contentLength,
                  ContentType = contentType.ToString()
              };
  
              if (contentLength &gt; maxContentLength || !contentType.Contains("text/html"))
              {
                  var page = BuildPage(null, loadTimeMeasure.ElapsedMilliseconds);
                  response.Content.Dispose();
                  return page;
              }
  
              var content = await response.Content.ReadAsStringAsync();
              var page2 = BuildPage(TryMapToHtmlNode(content), loadTimeMeasure.ElapsedMilliseconds);
              return page2;
          }
      }
      else
      {
          if (response.Content != null)
              response.Content.Dispose();
          return null;
      }
  }
  
  static HtmlNode TryMapToHtmlNode(string input)
  {
      try
      {
          var doc = new HtmlDocument();
          doc.LoadHtml(input);
  
          return doc.DocumentNode;
      }
      catch
      {
          return null;
      }
  }
  
  public static IEnumerable&lt;Uri&gt; ExtractLinks(this HtmlNode node, Uri baseUri)
  {
      var nodes = node.SelectNodes("//a[@href]");
  
      if (nodes == null || nodes.Count == 0)
          return Enumerable.Empty&lt;Uri&gt;();
  
      return nodes
          .Select(x =&gt; TryParseUri(x.GetAttributeValue("href", ""), baseUri))
          .Where(x =&gt; x != null &amp;&amp; (x.Scheme == "http" || x.Scheme == "https"))
          .Select(x =&gt;
          {
              var fragmentIndex = x.ToString().LastIndexOf("#", StringComparison.Ordinal);
              if (fragmentIndex == -1)
                  return x;
              return new Uri(x.ToString().Substring(0, fragmentIndex));
          });
  }
  
  static Uri TryParseUri(string url, Uri baseUri)
  {
      Uri newUri;
      if (Uri.TryCreate(baseUri, url, out newUri))
          return newUri;
      return null;
  }
  
  static ImmutableDictionary&lt;string, string&gt; ExtractHeaders(HttpResponseMessage response)
  {
      var headers = response.Headers
          .Select(x =&gt; Tuple.Create(x.Key, x.Value.First()));
  
      var contentHeaders = response.Content.Headers
          .Select(x =&gt; Tuple.Create(x.Key, x.Value.First()));
  
      return headers
          .Concat(contentHeaders)
          .ToImmutableDictionary(x =&gt; x.Item1, x =&gt; x.Item2);
  }
  </code></pre>
  
  <p>And:</p>
  
  <pre><code>static void Main(string[] args)
  {
      ServicePointManager.DefaultConnectionLimit = 5000;
      ServicePointManager.Expect100Continue = false;
  
      var hosts = File.ReadAllLines(@"C:\domains.txt");
  
      var handler = new WebRequestHandler
      {
          AllowAutoRedirect = false,
          UseProxy = false,
          Proxy = null
      };
  
      var client = new HttpClient
      {
          Timeout = TimeSpan.FromMinutes(20),
      };
  
      var semp = new SemaphoreSlim(10);
  
      foreach (var h in hosts)
      {
          semp.Wait();
  
          WebExtensions.GetPageAsync(client, 5242880, new Uri(h))
              .ContinueWith(_ =&gt; semp.Release());
      }
  
      while (true)
      {
          Console.ReadKey();
          GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced);
          GC.WaitForPendingFinalizers();
          GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced);
      }
  }
  </code></pre>