title: Web Scraping with WebBrowser and Single Thread Apartment C#
body: <p>This is a question about a slight variation on Noseratio's code in this question: [link]<a href="https://stackoverflow.com/questions/22239357/how-to-cancel-task-await-after-a-timeout-period/22262976#22262976">How to cancel Task await after a timeout period</a> </p>
  
  <p>I am able to build his code exactly as-is to create a Console application that returns the URL and OuterHtml of each of the three web pages specified in the code. However, when I put the same code in a WinForms application, the only output I get is </p>
  
  <p><code>URL:</code><br>
  <code>http://example.com</code></p>
  
  <p>That means that the code does not display the OuterHtml of the first page, and it does not display the URL or OuterHtml of either of the other two pages. What I would like to know is what about WinForms breaks the code and how to get around it.</p>
  
  <p>I expected that since the code still creates a new STA, which creates a new thread, it should not matter that I am using a WinForm. I am using VS2013 Ultimate, .NET 4.5.1.</p>
  
  <p>The WinForm consists of a single button. Clicking it is handled by <code>private void button1_Click(object sender, EventArgs e)</code>, which has a body identical to <code>Main</code> in Noseratio's code.</p>
  
  <pre><code>using System;
  using System.Collections.Generic;
  using System.ComponentModel;
  using System.Data;
  using System.Drawing;
  using System.Linq;
  using System.Text;
  using System.Threading;
  using System.Threading.Tasks;
  using System.Windows.Forms;
  using Microsoft.Win32;
  
  namespace WebScraperAsync005
  {
      public partial class Form1 : Form
      {
  
          // main logic
          static async Task ScrapSitesAsync(string[] urls, CancellationToken token)
          {
              using (var apartment = new MessageLoopApartment())
              {
                  // create WebBrowser inside MessageLoopApartment
                  var webBrowser = apartment.Invoke(() =&gt; new WebBrowser());
                  try
                  {
                      foreach (var url in urls)
                      {
                          Console.WriteLine("URL:\n" + url);
  
                          // cancel in 30s or when the main token is signalled
                          var navigationCts =  CancellationTokenSource.CreateLinkedTokenSource(token);
  
                            navigationCts.CancelAfter((int)TimeSpan.FromSeconds(30).TotalMilliseconds);
                          var navigationToken = navigationCts.Token;
  
                          // run the navigation task inside MessageLoopApartment
                          string html = await apartment.Run(() =&gt;
                          webBrowser.NavigateAsync(url, navigationToken), navigationToken);
  
                          Console.WriteLine("HTML:\n" + html);
                      }
                  }
                  finally
                  {
                      // dispose of WebBrowser inside MessageLoopApartment
                      apartment.Invoke(() =&gt; webBrowser.Dispose());
                  }
              }
          }
  
  
  
  
  
          public Form1()
          {
              InitializeComponent();
          }
  
          private void button1_Click(object sender, EventArgs e)
          {
              try
              {
                  WebBrowserExt.SetFeatureBrowserEmulation(); // enable HTML5
  
                  var cts = new   CancellationTokenSource((int)TimeSpan.FromMinutes(3).TotalMilliseconds);
  
                  var task = ScrapSitesAsync(
                      new[] { "http://example.com", "http://example.org",   "http://example.net" },
                      cts.Token);
  
                  task.Wait();
  
                  Console.WriteLine("Press Enter to exit...");
                  Console.ReadLine();
              }
              catch (Exception ex)
              {
                  while (ex is AggregateException &amp;&amp; ex.InnerException != null)
                      ex = ex.InnerException;
                  Console.WriteLine(ex.Message);
                  Environment.Exit(-1);
              }
          }
      }
  
  
  
      /// &lt;summary&gt;
      /// WebBrowserExt - WebBrowser extensions
      /// by Noseratio - https://stackoverflow.com/a/22262976/1768303
      /// &lt;/summary&gt;
      public static class WebBrowserExt
      {
          const int POLL_DELAY = 500;
  
          // navigate and download 
          public static async Task&lt;string&gt; NavigateAsync(this WebBrowser webBrowser,     string url, CancellationToken token)
          {
              // navigate and await DocumentCompleted
              var tcs = new TaskCompletionSource&lt;bool&gt;();
              WebBrowserDocumentCompletedEventHandler handler = (s, arg) =&gt;
              tcs.TrySetResult(true);
  
              using (token.Register(() =&gt; tcs.TrySetCanceled(), useSynchronizationContext: true))
              {
                  webBrowser.DocumentCompleted += handler;
                  try
                  {
                      webBrowser.Navigate(url);
                      await tcs.Task; // wait for DocumentCompleted
                  }
                  finally
                  {
                      webBrowser.DocumentCompleted -= handler;
                  }
              }
  
              // get the root element
              var documentElement = webBrowser.Document.GetElementsByTagName("html")[0];
  
              // poll the current HTML for changes asynchronosly
              var html = documentElement.OuterHtml;
              while (true)
              {
                  // wait asynchronously, this will throw if cancellation requested
                  await Task.Delay(POLL_DELAY, token);
  
                  // continue polling if the WebBrowser is still busy
                  if (webBrowser.IsBusy)
                      continue;
  
                  var htmlNow = documentElement.OuterHtml;
                  if (html == htmlNow)
                      break; // no changes detected, end the poll loop
  
                  html = htmlNow;
              }
  
              // consider the page fully rendered 
              token.ThrowIfCancellationRequested();
              return html;
          }
  
          // enable HTML5 (assuming we're running IE10+)
          // more info: https://stackoverflow.com/a/18333982/1768303
          public static void SetFeatureBrowserEmulation()
          {
              if (System.ComponentModel.LicenseManager.UsageMode !=     System.ComponentModel.LicenseUsageMode.Runtime)
                  return;
              var appName = System.IO.Path.GetFileName(System.Diagnostics.Process.GetCurrentProcess().MainModule.FileNa    me);
              Registry.SetValue(@"HKEY_CURRENT_USER\Software\Microsoft\Internet Explorer\Main\FeatureControl\FEATURE_BROWSER_EMULATION",
                  appName, 10000, RegistryValueKind.DWord);
          }
      }
  
      /// &lt;summary&gt;
      /// MessageLoopApartment
      /// STA thread with message pump for serial execution of tasks
      /// by Noseratio - https://stackoverflow.com/a/22262976/1768303
      /// &lt;/summary&gt;
      public class MessageLoopApartment : IDisposable
      {
          Thread _thread; // the STA thread
  
          TaskScheduler _taskScheduler; // the STA thread's task scheduler
  
          public TaskScheduler TaskScheduler { get { return _taskScheduler; } }
  
          /// &lt;summary&gt;MessageLoopApartment constructor&lt;/summary&gt;
          public MessageLoopApartment()
          {
              var tcs = new TaskCompletionSource&lt;TaskScheduler&gt;();
  
              // start an STA thread and gets a task scheduler
              _thread = new Thread(startArg =&gt;
              {
                  EventHandler idleHandler = null;
  
                  idleHandler = (s, e) =&gt;
                  {
                      // handle Application.Idle just once
                      Application.Idle -= idleHandler;
                      // return the task scheduler
                      tcs.SetResult(TaskScheduler.FromCurrentSynchronizationContext());
                  };
  
                  // handle Application.Idle just once
                  // to make sure we're inside the message loop
                  // and SynchronizationContext has been correctly installed
                  Application.Idle += idleHandler;
                  Application.Run();
              });
  
              _thread.SetApartmentState(ApartmentState.STA);
              _thread.IsBackground = true;
              _thread.Start();
              _taskScheduler = tcs.Task.Result;
          }
  
          /// &lt;summary&gt;shutdown the STA thread&lt;/summary&gt;
          public void Dispose()
          {
              if (_taskScheduler != null)
              {
                  var taskScheduler = _taskScheduler;
                  _taskScheduler = null;
  
                  // execute Application.ExitThread() on the STA thread
                  Task.Factory.StartNew(
                      () =&gt; Application.ExitThread(),
                      CancellationToken.None,
                      TaskCreationOptions.None,
                      taskScheduler).Wait();
  
                  _thread.Join();
                  _thread = null;
              }
          }
  
          /// &lt;summary&gt;Task.Factory.StartNew wrappers&lt;/summary&gt;
          public void Invoke(Action action)
          {
              Task.Factory.StartNew(action,
                  CancellationToken.None, TaskCreationOptions.None, _taskScheduler).Wait();
          }
  
          public TResult Invoke&lt;TResult&gt;(Func&lt;TResult&gt; action)
          {
              return Task.Factory.StartNew(action,
                  CancellationToken.None, TaskCreationOptions.None, _taskScheduler).Result;
          }
  
          public Task Run(Action action, CancellationToken token)
          {
              return Task.Factory.StartNew(action, token, TaskCreationOptions.None, _taskScheduler);
          }
  
          public Task&lt;TResult&gt; Run&lt;TResult&gt;(Func&lt;TResult&gt; action, CancellationToken token)
          {
              return Task.Factory.StartNew(action, token, TaskCreationOptions.None,     _taskScheduler);
          }
  
          public Task Run(Func&lt;Task&gt; action, CancellationToken token)
          {
              return Task.Factory.StartNew(action, token, TaskCreationOptions.None, _taskScheduler).Unwrap();
          }
  
          public Task&lt;TResult&gt; Run&lt;TResult&gt;(Func&lt;Task&lt;TResult&gt;&gt; action, CancellationToken token)
          {
              return Task.Factory.StartNew(action, token, TaskCreationOptions.None,     _taskScheduler).Unwrap();
          }
      }
  
  
  }
  </code></pre>