title: How can XPath be used to dig through (someone else's) poorly coded HTML?
body: <p>When I execute this C# code...</p>
  
  <pre><code>using System;
  using System.Collections.Generic;
  using System.ComponentModel;
  using System.Data;
  using System.Drawing;
  using System.Linq;
  using System.Text;
  using System.Text.RegularExpressions;
  using System.Threading.Tasks;
  using System.Windows.Forms;
  using System.IO;
  using System.Net;
  using System.Dynamic;
  using HtmlAgilityPack;
  
  namespace entropedizer
  {
      public partial class Form1 : Form
      {
          public Form1()
          {
              InitializeComponent();
          }
  
          public String postRequest(string url, string eventTarget)
          {
              // A "pre-request," sent to gather SessionID and POST data parameters for the main request
              HttpWebRequest prequest = (HttpWebRequest)WebRequest.Create("http://www.entropedia.info/Chart.aspx?chart=Chart");
              HttpWebResponse presponse = (HttpWebResponse)prequest.GetResponse();
              Stream pstream = presponse.GetResponseStream();
              StreamReader psr = new StreamReader(pstream);
              string phtml = psr.ReadToEnd();
  
              Match viewstate = Regex.Match(phtml, "id=\"__VIEWSTATE\".+/&gt;");
              Match eventvalidation = Regex.Match(phtml, "id=\"__EVENTVALIDATION\".+/&gt;");
              ASCIIEncoding encoding = new ASCIIEncoding();
              string postData = "__EVENTTARGET=" + eventTarget + "&amp;__VIEWSTATE=" + Uri.EscapeDataString(viewstate.ToString().Substring(24, viewstate.Length - 28)) + "&amp;__EVENTVALIDATION=" + Uri.EscapeDataString(eventvalidation.ToString().Substring(30, eventvalidation.Length - 34));
              byte[] data = encoding.GetBytes(postData);
  
              // The main request, intended to retreive the desired HTML
              HttpWebRequest request = (HttpWebRequest)WebRequest.Create("http://www.entropedia.info/Chart.aspx?chart=Chart");
              request.Method = "POST";
              request.ContentType = "application/x-www-form-urlencoded";
  
              request.CookieContainer = new CookieContainer();
              Cookie sessionId = new Cookie("ASP.NET_SessionId", Regex.Match(presponse.Headers.ToString(), "ASP.NET_SessionId=.+ d").ToString().Substring(18, Regex.Match(presponse.Headers.ToString(), "ASP.NET_SessionId=.+ d").Length - 21), "/", ".entropedia.info");
              request.CookieContainer.Add(new Uri("http://www.entropedia.info/Chart.aspx?chart=Chart"), sessionId);
  
              Stream stream = request.GetRequestStream();
              stream.Write(data, 0, data.Length);
              stream.Close();
  
              HttpWebResponse response = (HttpWebResponse)request.GetResponse();
              stream = response.GetResponseStream();
  
              StreamReader sr = new StreamReader(stream);
  
              return sr.ReadToEnd();
          }
  
          private void Form1_Load(object sender, EventArgs e)
          {
              System.Net.ServicePointManager.Expect100Continue = false;
              HtmlAgilityPack.HtmlDocument hChart = new HtmlAgilityPack.HtmlDocument();
              hChart.LoadHtml(postRequest("http://www.entropedia.info/Chart.aspx?chart=Chart", "ctl00%24ContentPlaceHolder1%24DG1%24ctl19%24ctl05"));
  
              HtmlNodeCollection chartStrings = hChart.DocumentNode.SelectNodes("/");
  
              if (chartStrings != null)
              {
                  foreach (HtmlNode i in chartStrings)
                  {
                      System.IO.File.WriteAllText("C:/Users/Admin/Desktop/WholeDocument.txt", i.OuterHtml);
                  }
              }
              else
              {
                  MessageBox.Show("Error:  Null item list.");
              }
          }
      }
  }
  </code></pre>
  
  <p>...the following HTML is written to a text file.</p>
  
  <p><a href="http://pastebin.com/FALerBWR" rel="nofollow">http://pastebin.com/FALerBWR</a></p>
  
  <p>When I change the line in my C# code to HtmlNodeCollection <code>chartStrings = hChart.DocumentNode.SelectNodes("/html/body");</code> the 400+ lines of HTML within the body is written to the text file instead.</p>
  
  <p>When I change the line to <code>HtmlNodeCollection chartStrings = hChart.DocumentNode.SelectNodes("/html/body/form");</code> only a single line of code (the form opening tag with its attributes) is written to the text file.  It should write many lines (most of the document).  I believe <code>HtmlAgilityPack</code> becomes confused due to the malformed HTML tags.  Is there a way to programatically work around this?  I do not want to correct the HTML manually every time I run the program! </p>