How do i search over the List<string> and check if the web address already exist or not?
<p>In this case its not working good it keep adding the same links to the List.</p>

<pre><code>using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using HtmlAgilityPack;
using System.IO;
using System.Text.RegularExpressions;
using System.Xml.Linq;
using System.Net;
using System.Web;


namespace GatherLinks
{
    public partial class Form1 : Form
    {
        int sites = 0;
        int y = 0;
        string url = @"http://www.google.co.il";
        string guys = "http://www.google.com";

        public Form1()
        {
            InitializeComponent();

            List&lt;string&gt; a = webCrawler(guys, 2);
        }

        private void Form1_Load(object sender, EventArgs e)
        {

        }

        private int factorial(int n)
        {
            if (n == 0) return 1;
            else y = n * factorial(n - 1);
            richTextBox1.Text = y.ToString();
            return y;


        }

        private List&lt;string&gt; getLinks(HtmlAgilityPack.HtmlDocument document)
        {

            List&lt;string&gt; mainLinks = new List&lt;string&gt;();
            var linkNodes = document.DocumentNode.SelectNodes("//a[@href]");
            if (linkNodes != null)
            {
                foreach (HtmlNode link in linkNodes)
                {
                    var href = link.Attributes["href"].Value;
                    mainLinks.Add(href);
                }
            }
            return mainLinks;

        }


        private List&lt;string&gt; webCrawler(string url, int levels)
        {

                HtmlAgilityPack.HtmlDocument doc;
                HtmlWeb hw = new HtmlWeb();
                List&lt;string&gt; webSites;// = new List&lt;string&gt;();
                List&lt;string&gt; csFiles = new List&lt;string&gt;();

                csFiles.Add("temp string to know that something is happening in level = " + levels.ToString());
                csFiles.Add("current site name in this level is : " + url);
                                try
                {
                    doc = hw.Load(url);
                    webSites = getLinks(doc);

                    if (levels == 0)
                    {
                        return csFiles;
                    }
                    else
                    {
                        int actual_sites = 0;
                        for (int i = 0; i &lt; webSites.Count() &amp;&amp; i &lt; 20; i++)                         {
                            string t = webSites[i];
                                                        if ((t.StartsWith("http://") == true) || (t.StartsWith("https://") == true)) // replace this with future FilterJunkLinks function
                            {
                               // for (int e = 0; e &lt; csFiles.Count; e++)
                               // {
                                    if (csFiles.Contains(t))
                                    {
                                    }
                                    else
                                    {
                                        actual_sites++;
                                        csFiles.AddRange(webCrawler(t, levels - 1));
                                        Texts(richTextBox1, "Level Number " + levels + " " + t + Environment.NewLine, Color.Red);
                                    }
                               // }
                            }
                        }
                        // report to a message box only at high levels..
                        //if (levels==1)
                        //MessageBox.Show(actual_sites.ToString());

                        return csFiles;
                    }



                }
                catch
                {
                    return csFiles;
                }

        }
</code></pre>

<p>And the Texts function:</p>

<pre><code>public void Texts(RichTextBox box, string text, Color color)
        {
            box.SelectionStart = box.TextLength;
            box.SelectionLength = 0;

            box.SelectionColor = color;
            box.AppendText(text);
            box.SelectionColor = box.ForeColor;
        } 
</code></pre>

<p>I need two things to do in the webCrawler function:</p>

<ol>
<li><p>If the url variable can not be resolved then the try and catch should do the job.</p></li>
<li><p>If the List csFiles contains already the same items do not add them again. For example if in the csFiles there already <a href="http://www.google.com" rel="nofollow">http://www.google.com</a> then do not add <a href="http://www.google.com" rel="nofollow">http://www.google.com</a> again so in the end the csFiles List will contain <a href="http://www.google.com" rel="nofollow">http://www.google.com</a> only once.</p></li>
</ol>
