title: Web crawler in NodeJS - Encrypted page or something like
body: <p>I'm trying to extract some data from a website, so I wrote kind of a web crawler in NodeJS.</p>
  
  <p>The problem is that the page is returning the HTML encoded, like this:</p>
  
  <pre><code>&lt;/�◄♣b∟�♣66c���▬�iǌi�g��-��~\��r&gt;
  </code></pre>
  
  <p>What is this and how can I "fix it"?</p>
  
  <h2>UPDATE</h2>
  
  <pre><code>var http = require('follow-redirects').http;
  
  // Utility function that downloads a URL and invokes
  // callback with the data.
  function download(url, callback) {
  
      var request_options = {
          host: url,
          headers: {
              'user-agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36'
          },
          path: '/'
      };
  
      http.get(url, function (res) {
          var data = "";
          res.on('data', function (chunk) {
              data += chunk;
          });
          res.on("end", function () {
              callback(data);
          });
      }).on("error", function () {
          callback(null);
      });
  }
  
  var cheerio = require("cheerio");
  
  var baseUrl = "http://www.mensagemaniversario.com.br";
  
  var fs = require("fs");
  
  var Entities = require('html-entities').AllHtmlEntities;
  
  entities = new Entities();
  
  var page = 1;
  
  function recursiveDownload(url) {
      download(url, function (data) {
          if (data) {
              // console.log(data);
              var $ = cheerio.load(data);
              console.log($.html());
              $("div.box-list &gt; p").each(function (i, e) {
                  var frase = $(e);
  
  
                  fs.appendFile("dados.xml", "&lt;frase&gt;" + entities.decode(frase.html()) + "&lt;/frase&gt;");
              });
  
              console.log("Processado página " + url);
              var linkNextPage = $("div.pagination a:last-child");
              if (linkNextPage.length &gt; 0) {
                  page++;
                  recursiveDownload(baseUrl + linkNextPage.attr("href"));
              }
          }
      });
  }
  
  recursiveDownload(baseUrl + '/15-anos');
  </code></pre>