title: Reading from a HttpResponseStream fails
body: <p>I'm running into an issue where reading from a HttpResponseStream fails because the StreamReader that I'm wrapping around in reads faster that the Response Stream gets the actual response.  I'm retrieving a reasonably small sized file (around 60k) but the Parser which processes the response into an actual object fails because it hits an unexpected character (Code 65535) which from experience I know to be the character produced when you read from a StreamReader and there are no further characters available.</p>
  
  <p>For the record I know that the content being returned is valid and will parse correctly since the failure occurs at different points in the file each time I run the code.  It's the parser.Load() line in the following where it fails.</p>
  
  <p>Is there a way to ensure I've read all the content before attempting to parse it short of copying the response stream into a MemoryStream or string and then processing it?</p>
  
  <pre><code>    /// &lt;summary&gt;
      /// Makes a Query where the expected Result is an RDF Graph ie. CONSTRUCT and DESCRIBE Queries
      /// &lt;/summary&gt;
      /// &lt;param name="sparqlQuery"&gt;SPARQL Query String&lt;/param&gt;
      /// &lt;returns&gt;RDF Graph&lt;/returns&gt;
      public Graph QueryWithResultGraph(String sparqlQuery)
      {
          try
          {
              //Build the Query URI
              StringBuilder queryUri = new StringBuilder();
              queryUri.Append(this._endpoint.ToString());
              queryUri.Append("?query=");
              queryUri.Append(Uri.EscapeDataString(sparqlQuery));
  
              if (!this._defaultGraphUri.Equals(String.Empty))
              {
                  queryUri.Append("&amp;default-graph-uri=");
                  queryUri.Append(Uri.EscapeUriString(this._defaultGraphUri));
              }
  
              //Make the Query via HTTP
              HttpWebResponse httpResponse = this.DoQuery(new Uri(queryUri.ToString()),false);
  
              //Set up an Empty Graph ready
              Graph g = new Graph();
              g.BaseURI = this._endpoint;
  
              //Parse into a Graph based on Content Type
              String ctype = httpResponse.ContentType;
              IRDFReader parser = MIMETypesHelper.GetParser(ctype);
              parser.Load(g, new StreamReader(httpResponse.GetResponseStream()));
  
              return g;
          }
          catch (UriFormatException uriEx)
          {
              //URI Format Invalid
              throw new Exception("The format of the URI was invalid", uriEx);
          }
          catch (WebException webEx)
          {
              //Some sort of HTTP Error occurred
              throw new Exception("A HTTP Error occurred", webEx);
          }
          catch (RDFException)
          {
              //Some problem with the RDF or Parsing thereof
              throw;
          }
          catch (Exception)
          {
              //Other Exception
              throw;
          }
      }
  
      /// &lt;summary&gt;
      /// Internal Helper Method which executes the HTTP Requests against the SPARQL Endpoint
      /// &lt;/summary&gt;
      /// &lt;param name="target"&gt;URI to make Request to&lt;/param&gt;
      /// &lt;param name="sparqlOnly"&gt;Indicates if only SPARQL Result Sets should be accepted&lt;/param&gt;
      /// &lt;returns&gt;HTTP Response&lt;/returns&gt;
      private HttpWebResponse DoQuery(Uri target, bool sparqlOnly)
      {
          //Expect errors in this function to be handled by the calling function
  
          //Set-up the Request
          HttpWebRequest httpRequest;
          HttpWebResponse httpResponse;
          httpRequest = (HttpWebRequest)WebRequest.Create(target);
  
          //Use HTTP GET/POST according to user set preference
          if (!sparqlOnly)
          {
              httpRequest.Accept = MIMETypesHelper.HTTPAcceptHeader();
              //For the time being drop the application/json as this doesn't play nice with Virtuoso
              httpRequest.Accept = httpRequest.Accept.Replace("," + MIMETypesHelper.JSON[0], String.Empty);
          }
          else
          {
              httpRequest.Accept = MIMETypesHelper.HTTPSPARQLAcceptHeader();
          }
          httpRequest.Method = this._httpMode;
          httpRequest.Timeout = this._timeout;
  
          //HTTP Debugging
          if (Options.HTTPDebugging)
          {
              Tools.HTTPDebugRequest(httpRequest);
          }
  
          httpResponse = (HttpWebResponse)httpRequest.GetResponse();
  
          //HTTP Debugging
          if (Options.HTTPDebugging)
          {
              Tools.HTTPDebugResponse(httpResponse);
          }
  
          return httpResponse;
      }
  </code></pre>
  
  <p><strong>Edit</strong></p>
  
  <p>To clarify what I already stated this is <em>not</em> a bug in the Parser, this is an issue of the StreamReader reading faster than the Response Stream provides data.  I can get around this by doing the following but would like suggestions of better or more elegant solutions:</p>
  
  <pre><code>            //Parse into a Graph based on Content Type
              String ctype = httpResponse.ContentType;
              IRDFReader parser = MIMETypesHelper.GetParser(ctype);
              Stream response = httpResponse.GetResponseStream();
              MemoryStream temp = new MemoryStream();
              Tools.StreamCopy(response, temp);
              response.Close();
              temp.Seek(0, SeekOrigin.Begin);
              parser.Load(g, new StreamReader(temp));
  </code></pre>
  
  <p><strong>Edit 2</strong></p>
  
  <p>BlockingStreamReader class as per Eamon's suggestion:</p>
  
  <pre><code>/// &lt;summary&gt;
  /// A wrapper to a Stream which does all its Read() and Peek() calls using ReadBlock() to handle slow underlying streams (eg Network Streams)
  /// &lt;/summary&gt;
  public sealed class BlockingStreamReader : StreamReader
  {
      private bool _peeked = false;
      private int _peekChar = -1;
  
      public BlockingStreamReader(StreamReader reader) : base(reader.BaseStream) { }
  
      public BlockingStreamReader(Stream stream) : base(stream) { }
  
      public override int Read()
      {
          if (this._peeked)
          {
              this._peeked = false;
              return this._peekChar;
          }
          else
          {
              if (this.EndOfStream) return -1;
  
              char[] cs = new char[1];
              base.ReadBlock(cs, 0, 1);
  
              return cs[0];
          }
      }
  
      public override int Peek()
      {
          if (this._peeked)
          {
              return this._peekChar;
          }
          else
          {
              if (this.EndOfStream) return -1;
  
              this._peeked = true;
  
              char[] cs = new char[1];
              base.ReadBlock(cs, 0, 1);
  
              this._peekChar = cs[0];
              return this._peekChar;
          }
      }
  
      public new bool EndOfStream
      {
          get
          {
              return (base.EndOfStream &amp;&amp; !this._peeked);
          }
      }
  }
  </code></pre>
  
  <p><strong>Edit 3</strong></p>
  
  <p>Here is a much improved solution which can wrap any <code>TextReader</code> and provide an <code>EndOfStream</code> property.  It uses an internal buffer which is filled by using <code>ReadBlock()</code> on the wrapped <code>TextReader</code>.  All the Read() methods of the reader can the be defined using this buffer, buffer size is configurable:</p>
  
  <pre><code>    /// &lt;summary&gt;
  /// The BlockingTextReader is an implementation of a &lt;see cref="TextReader"&gt;TextReader&lt;/see&gt; designed to wrap other readers which may or may not have high latency.
  /// &lt;/summary&gt;
  /// &lt;remarks&gt;
  /// &lt;para&gt;
  /// This is designed to avoid premature detection of end of input when the input has high latency and the consumer tries to read from the input faster than it can return data.  All methods are defined by using an internal buffer which is filled using the &lt;see cref="TextReader.ReadBlock"&gt;ReadBlock()&lt;/see&gt; method of the underlying &lt;see cref="TextReader"&gt;TextReader&lt;/see&gt;
  /// &lt;/para&gt;
  /// &lt;/remarks&gt;
  public sealed class BlockingTextReader : TextReader
  {
      private char[] _buffer;
      private int _pos = -1;
      private int _bufferAmount = -1;
      private bool _finished = false;
      private TextReader _reader;
  
      public const int DefaultBufferSize = 1024;
  
      public BlockingTextReader(TextReader reader, int bufferSize)
      {
          if (reader == null) throw new ArgumentNullException("reader", "Cannot read from a null TextReader");
          if (bufferSize &lt; 1) throw new ArgumentException("bufferSize must be &gt;= 1", "bufferSize");
          this._reader = reader;
          this._buffer = new char[bufferSize];
      }
  
      public BlockingTextReader(TextReader reader)
          : this(reader, DefaultBufferSize) { }
  
      public BlockingTextReader(Stream input, int bufferSize)
          : this(new StreamReader(input), bufferSize) { }
  
      public BlockingTextReader(Stream input)
          : this(new StreamReader(input)) { }
  
      private void FillBuffer()
      {
          this._pos = -1;
          if (this._finished)
          {
              this._bufferAmount = 0;
          }
          else
          {
              this._bufferAmount = this._reader.ReadBlock(this._buffer, 0, this._buffer.Length);
              if (this._bufferAmount == 0 || this._bufferAmount &lt; this._buffer.Length) this._finished = true;
          }
      }
  
      public override int ReadBlock(char[] buffer, int index, int count)
      {
          if (count == 0) return 0;
          if (buffer == null) throw new ArgumentNullException("buffer");
          if (index &lt; 0) throw new ArgumentException("index", "Index must be &gt;= 0");
          if (count &lt; 0) throw new ArgumentException("count", "Count must be &gt;= 0");
          if ((buffer.Length - index) &lt; count) throw new ArgumentException("Buffer too small");
  
          if (this._bufferAmount == -1 || this._pos &gt;= this._bufferAmount)
          {
              if (!this._finished)
              {
                  this.FillBuffer();
                  if (this.EndOfStream) return 0;
              }
              else
              {
                  return 0;
              }
          }
  
          this._pos = Math.Max(0, this._pos);
          if (count &lt;= this._bufferAmount - this._pos)
          {
              //If we have sufficient things buffered to fufill the request just copy the relevant stuff across
              Array.Copy(this._buffer, this._pos, buffer, index, count);
              this._pos += count;
              return count;
          }
          else
          {
              int copied = 0;
              while (copied &lt; count)
              {
                  int available = this._bufferAmount - this._pos;
                  if (count &lt; copied + available)
                  {
                      //We can finish fufilling this request this round
                      int toCopy = Math.Min(available, count - copied);
                      Array.Copy(this._buffer, this._pos, buffer, index + copied, toCopy);
                      copied += toCopy;
                      this._pos += toCopy;
                      return copied;
                  }
                  else
                  {
                      //Copy everything we currently have available
                      Array.Copy(this._buffer, this._pos, buffer, index + copied, available);
                      copied += available;
                      this._pos = this._bufferAmount;
  
                      if (!this._finished)
                      {
                          //If we haven't reached the end of the input refill our buffer and continue
                          this.FillBuffer();
                          if (this.EndOfStream) return copied;
                          this._pos = 0;
                      }
                      else
                      {
                          //Otherwise we have reached the end of the input so just return what we've managed to copy
                          return copied;
                      }
                  }
              }
              return copied;
          }
      }
  
      public override int Read(char[] buffer, int index, int count)
      {
          return this.ReadBlock(buffer, index, count);
      }
  
      public override int Read()
      {
          if (this._bufferAmount == -1 || this._pos &gt;= this._bufferAmount - 1)
          {
              if (!this._finished)
              {
                  this.FillBuffer();
                  if (this.EndOfStream) return -1;
              }
              else
              {
                  return -1;
              }
          }
  
          this._pos++;
          return (int)this._buffer[this._pos];
      }
  
      public override int Peek()
      {
          if (this._bufferAmount == -1 || this._pos &gt;= this._bufferAmount - 1)
          {
              if (!this._finished)
              {
                  this.FillBuffer();
                  if (this.EndOfStream) return -1;
              }
              else
              {
                  return -1;
              }
          }
  
          return (int)this._buffer[this._pos + 1];
      }
  
      public bool EndOfStream
      {
          get
          {
              return this._finished &amp;&amp; (this._pos &gt;= this._bufferAmount - 1);
          }
      }
  
      public override void Close()
      {
          this._reader.Close();
      }
  
      protected override void Dispose(bool disposing)
      {
          this.Close();
          this._reader.Dispose();
          base.Dispose(disposing);
      }
  }
  </code></pre>