How do i check for url timeout ? Using WebClient and HtmlAgilityPack
<p>I have this function im using WebClient to check if the url is a content type and after that i have another function and some properties im using to check for timeout if the url for example is not exist or busy or down so i want to show the user that after X seconds its timed out. The part of the time out is not working. The function getHtmlDocumentWebClient is working:</p>

<pre><code>using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using HtmlAgilityPack;
using System.IO;
using System.Text.RegularExpressions;
using System.Xml.Linq;
using System.Net;
using System.Web;
using System.Threading;
using DannyGeneral;

namespace GatherLinks
{
    class TimeOut
    {
        private Form1 frm1reference;

        public TimeOut(Form1 frm1)
        {
            frm1reference = frm1;
        }

        class MyClient : WebClient
        {
            public bool HeadOnly { get; set; }
            protected override WebRequest GetWebRequest(Uri address)
            {
                WebRequest req = base.GetWebRequest(address);
                if (HeadOnly &amp;&amp; req.Method == "GET")
                {
                    req.Method = "HEAD";
                }
                return req;
            }
        }

        private static HtmlAgilityPack.HtmlDocument getHtmlDocumentWebClient(string url, bool useProxy, string proxyIp, int proxyPort, string usename, string password)
        {
            //**************************************\\
            HtmlAgilityPack.HtmlDocument doc = null;
            using (MyClient clients = new MyClient())
            {
                clients.HeadOnly = true;
                //string uri = "http://www.google.com";
                byte[] body = clients.DownloadData(url);
                // note should be 0-length
                string type = clients.ResponseHeaders["content-type"];
                clients.HeadOnly = false;
                // check 'tis not binary... we'll use text/, but could
                // check for text/html
                if (type.StartsWith(@"text/html"))
                {
                    string text = clients.DownloadString(url);

                    try
                    {
                        doc = new HtmlAgilityPack.HtmlDocument();
                        WebClient client = new WebClient();
                        //client.Headers.Add("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)");
                        client.Credentials = CredentialCache.DefaultCredentials;
                        client.Proxy = WebRequest.DefaultWebProxy;
                        if (useProxy)
                        {
                            //Proxy                
                            if (!string.IsNullOrEmpty(proxyIp))
                            {
                                WebProxy p = new WebProxy(proxyIp, proxyPort);
                                if (!string.IsNullOrEmpty(usename))
                                {
                                    if (password == null)
                                        password = string.Empty;
                                    NetworkCredential nc = new NetworkCredential(usename, password);
                                    p.Credentials = nc;
                                }
                            }
                        }
                        doc.Load(client.OpenRead(url));
                    }
                    catch (Exception e)
                    {

                    }
                }

            }
            if (doc == null)
            {
                MessageBox.Show("Doc is null   " + doc + " The link that did it was    " + url);
            }
            return doc;
        }

        public HtmlAgilityPack.HtmlDocument GetHtmlDoc(string url, Options options, ProxyModel proxy)
        {
            try
            {
                //Wait before execute 

                if (options != null)
                {
                    if (options.Request_pause_milliseconds &gt;= 0)
                        System.Threading.Thread.Sleep(options.Request_pause_milliseconds);
                }
                //Execute WITH Proxy        
                if (proxy != null)
                {
                    //if proxy ip is empty use standard request
                    if (proxy.Ip.ToLower() != "")
                        return getHtmlDocumentWebClient(url, true, proxy.Ip, Int32.Parse(proxy.Port), proxy.Username, proxy.Password);
                    else return getHtmlDocumentWebClient(url, false, string.Empty, 0, string.Empty, string.Empty);
                }
                else return getHtmlDocumentWebClient(url, false, string.Empty, 0, string.Empty, string.Empty);
            }
            catch (TimeoutException tEx)
            {
                System.Threading.Thread.Sleep(options.Pause_after_timeout_milliseconds);
                Logger.Write(string.Format("TimeoutException : Noc.HtmlScraper.Request.Load.GetHtmlDoc({0},options,proxies)" + url + "  " + tEx));
                frm1reference.timeOut = true;
                return null;
            }
            catch (Exception ex)
            {
                Logger.Write("TimeoutException : Noc.HtmlScraper.Request.Load.GetHtmlDoc({0},options,proxies)" + url + "  " + ex);
                frm1reference.timeOut = true;
                return null;
            }
        }

        public class Options
        {
            public int Request_pause_milliseconds { get; set; }
            public int Pause_after_timeout_milliseconds { get; set; }
            public Options() { }
            public Options(int request_pause_milliseconds, int Pause_after_timeout_milliseconds, bool EnableProxy)
            {
                this.Request_pause_milliseconds = request_pause_milliseconds;
                this.Pause_after_timeout_milliseconds = Pause_after_timeout_milliseconds;
            }
        }
        public class ProxyModel
        {
            public string Ip { get; set; }
            public string Port { get; set; }
            public string Username { get; set; }
            public string Password { get; set; }
            public ProxyModel() { }
            public ProxyModel(string Ip, string Port, string Username, string Password)
            {
                this.Ip = Ip; this.Username = Username; this.Password = Password; this.Port = Port;
            }
        }
    }
}
</code></pre>

<p>In Form1 im using it as:</p>

<pre><code>using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using HtmlAgilityPack;
using System.IO;
using System.Text.RegularExpressions;
using System.Xml.Linq;
using System.Net;
using System.Web;
using System.Threading;
using DannyGeneral;
using GatherLinks;



namespace GatherLinks
{
    public partial class Form1 : Form
    {
        RetrieveWebContent webContent;
        public bool timeOut;
        TimeOut to;
        TimeOut.Options reqOptions;
        bool removeExt;
        bool buttonSwitch;
        ToolTip tip1;
        int levelsToCrawl;
        bool downLoadImages;
        ChangeLink cl;
        List&lt;string&gt; currentCrawlingSite;
        List&lt;string&gt; sitesToCrawl;
        int actual_sites;
        int y = 0;
        string mainUrl = "";

        public Form1()
        {
            InitializeComponent();

            tip1 = new ToolTip();
            mainUrl =  OptionsDB.get_changedWebSite();
            checkBox1.Checked = OptionsDB.Get_begin_imagesDownloads();
            if (checkBox1.Checked == true)
            {
                downLoadImages = true;
            }
            else
            {
                downLoadImages = false;
            }
            checkBox2.Checked = OptionsDB.Get_localOnly();
            if (checkBox2.Checked == true)
            {
                removeExt = true;
            }
            else
            {
                removeExt = false;
            }
            numericUpDown1.Value = OptionsDB.Get_num1_value();
            webContent = new RetrieveWebContent();
            to = new TimeOut(this);
            button3.Enabled = false;
            timeOut = false;
            reqOptions = new TimeOut.Options(80, 10000, false);
            buttonSwitch = true;
            levelsToCrawl = (int)numericUpDown1.Value;
            Logger.exist();
            currentCrawlingSite = new List&lt;string&gt;();
            sitesToCrawl = new List&lt;string&gt;();
            actual_sites = 0;
            label6.Visible = false;
            label4.Text = mainUrl;
            label7.Text = "0";
            label3.Text = "0";
        }

        private void Form1_Load(object sender, EventArgs e)
        {

        }



        private List&lt;string&gt; getLinks(HtmlAgilityPack.HtmlDocument document)
        {

                List&lt;string&gt; mainLinks = new List&lt;string&gt;();
                var linkNodes = document.DocumentNode.SelectNodes("//a[@href]");
                if (linkNodes != null)
                {
                    foreach (HtmlNode link in linkNodes)
                    {
                        var href = link.Attributes["href"].Value;
                        if (href.StartsWith("http://") == true || href.StartsWith("https://") == true || href.StartsWith("www") == true) // filter for http 
                        {
                            mainLinks.Add(href);
                        }
                    }
                }

                return mainLinks;

        }


        private List&lt;string&gt; test(string url, int levels,DoWorkEventArgs eve)
        {

            levels = levelsToCrawl;
            HtmlWeb hw = new HtmlWeb();
            List&lt;string&gt; webSites;
            List&lt;string&gt; csFiles = new List&lt;string&gt;();

            csFiles.Add("temp string to know that something is happening in level = " + levels.ToString());
            csFiles.Add("current site name in this level is : " + url);
            /* later should be replaced with real cs files .. cs files links..*/
            try
            {
                this.Invoke(new MethodInvoker(delegate { Texts(richTextBox1, "Loading The Url:   " , Color.Red); }));
                this.Invoke(new MethodInvoker(delegate { Texts(richTextBox1, url + "...",Color.Blue); }));
                HtmlAgilityPack.HtmlDocument doc =  to.GetHtmlDoc(url, reqOptions, null);


                if (timeOut == true)
                {
                    this.Invoke(new MethodInvoker(delegate { Texts(richTextBox1, " There Was A TimeOut" + Environment.NewLine , Color.Green); }));
                    timeOut = false;
</code></pre>

<p>The problem is that the timeout part is not working. Tried in Form1 to use local bool variable wich is public and use it in the new class also to check/get when its time out but its not working as i wanted it to be.</p>

<p>The part of the content type checking is working !!!</p>

<p>The part of the time out in the new class is not working. How should i fix it ?</p>
