title: Getting 302 status and too many redirects for valid URL while using httpWebRequest and WebClient class
body: <p>Hi guys need help i am trying to make a console application which checks whether a website is available or not. Also i am trying to get the title of the page.
  For doing this i am using HttpWebRequest Class (for getting status) and WebClient class (for getting title).</p>
  
  <p>Note: The page i am trying to get is on a private server.
  URl format is <strong>(applicationname-environment.corporation.companyname.com)</strong>
  <strong>example: FIFO-dev.corp.tryit.com</strong></p>
  
  <p>When i try to get the status it is always giving me 401 as its status even though the page is up and running</p>
  
  <pre><code>List&lt;int&gt; Web_Status = new List&lt;int&gt;();
  foreach (var URL in WEB_URL)
  {
      try
      {
          HttpWebRequest Web_Test = (HttpWebRequest)WebRequest.Create("http://" + URL);
          Web_Test.AllowAutoRedirect = true;
          HttpWebResponse Web_response = (HttpWebResponse)Web_Test.GetResponse();
          Web_Status.Add((int)Web_response.StatusCode);
          Web_response.Close();
      }
      catch (System.Net.WebException ex)
      {
          HttpWebResponse Web_response = (HttpWebResponse)ex.Response;
          Web_Status.Add((int)Web_response.StatusCode);
      }
  }
  </code></pre>
  
  <p><strong>Also note while giving url's i am making sure that i am not reentering http://.</strong></p>
  
  <p>The below code is giving this error</p>
  
  <blockquote>
    <p>"System.Net.WebException: The remote server returned an error: (401)
    Unauthorized.</p>
    
    <p>at System.Net.WebClient.DownloadDataInternal(Uri address,
    WebRequest&amp; request)</p>
    
    <p>at System.Net.WebClient.DownloadString(Uri address)</p>
    
    <p>at website_monitoring.Get_Title.Title(List`1 WEB_URL) in "</p>
  </blockquote>
  
  <pre><code>string source = "";
  List&lt;string&gt; status = new List&lt;string&gt;();
  WebClient x = new WebClient();
  foreach (var item in WEB_URL)
  {
      try
      {
          source = x.DownloadString("http://" + item);
          status.Add(Regex.Match(source, @"\&lt;title\b[^&gt;]*\&gt;\s*(?&lt;Title&gt;[\s\S]*?)\&lt;/title\&gt;", RegexOptions.IgnoreCase).Groups["Title"].Value);
      }
      catch (System.Net.WebException ex)
      {
          status.Add(ex.ToString());
      }
  }
  </code></pre>
  
  <p><strong>Sorry guys i cant give the exact url i am trying on.</strong></p>
  
  <hr>
  
  <blockquote>
    <p>This code is working with all the common websites and blogs like
    "stackoverflow.com","<a href="http://understandingarduino.blogspot.in/" rel="nofollow noreferrer">http://understandingarduino.blogspot.in/</a>" and so
    on.</p>
  </blockquote>
  
  <hr>
  
  <p>Update 1: following mammago suggestion i was able to handle 4xx issue but now it is giving too many redirects issues while getting title.
  i was able to handle 302 status issue by autoredirect property to 1000;</p>
  
  <pre><code>List&lt;int&gt; Web_Status = new List&lt;int&gt;();
          foreach (var URL in WEB_URL)
          {
              try
              {
                  HttpWebRequest Web_Test = (HttpWebRequest)WebRequest.Create("http://" + URL);
  
                  // Set credentials to use for this request.
                  Web_Test.Credentials = CredentialCache.DefaultCredentials;
  
                  Web_Test.CookieContainer = new CookieContainer();
                  Web_Test.AllowAutoRedirect = true;
                  Web_Test.MaximumAutomaticRedirections = 1000;
                  //Web_Test.UserAgent =
                  HttpWebResponse Web_response = (HttpWebResponse)Web_Test.GetResponse();
                  Web_Status.Add((int)Web_response.StatusCode);
                  Web_response.Close();
              }
              catch (System.Net.WebException ex)
              {
                  HttpWebResponse Web_response = (HttpWebResponse)ex.Response;
                  Web_Status.Add((int)Web_response.StatusCode);
              }
          }
  </code></pre>
  
  <p>Now all i need help is how to handle auto- redirect issue in this segment</p>
  
  <pre><code>string source = "";
          List&lt;string&gt; status = new List&lt;string&gt;();
          WebClient x = new WebClient();
  
          //letting the website know its a known user who is accessing it (as if website have auto authentication)
          x.Credentials = CredentialCache.DefaultCredentials;
          foreach (var item in WEB_URL)
          {
              try
              {
                  source = x.DownloadString("http://" + item);
                  status.Add(Regex.Match(source, @"\&lt;title\b[^&gt;]*\&gt;\s*(?&lt;Title&gt;[\s\S]*?)\&lt;/title\&gt;", RegexOptions.IgnoreCase).Groups["Title"].Value);
              }
              catch (System.Net.WebException ex)
              {
                  status.Add(ex.ToString());
                  //source = x.DownloadString("http://" + ex);
                  //status.Add(Regex.Match(source, @"\&lt;title\b[^&gt;]*\&gt;\s*(?&lt;Title&gt;[\s\S]*?)\&lt;/title\&gt;", RegexOptions.IgnoreCase).Groups["Title"].Value);
              }
          }
  </code></pre>
  
  <blockquote>
    <p>"System.Net.WebException: Too many automatic redirections were
    attempted.    at System.Net.WebClient.DownloadDataInternal(Uri
    address, WebRequest&amp; request)    at
    System.Net.WebClient.DownloadString(Uri address)</p>
  </blockquote>