text: var URL = require('url'),
      util = require('util'),
      events = require('events'),
      Buffer = require('buffer').Buffer,
      fs = require('fs'),
      path = require('path'),
      jsdom = require('jsdom'),
      uglifyjs = require('uglify-js'),
      jsp = uglifyjs.parser,
      pro = uglifyjs.uglify,
      compress = null, // import only when required - might make the command line tool easier to use
      http = {
        http: require('http'),
        https: require('https')
      };
  
  function compressCSS(css) {
    return css
      .replace(/\s+/g, ' ')
      .replace(/:\s+/g, ':')
      .replace(/\/\*.*?\*\//g, '')
      .replace(/\} /g, '}')
      .replace(/ \{/g, '{')
      // .replace(/\{ /g, '{')
      .replace(/; /g, ';')
      .replace(/\n+/g, '');
  }
  
  function removeComments(element) {
    if (!element || !element.childNodes) return;
    var nodes = element.childNodes,
        i = nodes.length;
    
    while (i--) {
      if (nodes[i].nodeName === '#comment' && nodes[i].nodeValue.indexOf('[') !== 0) {
        element.removeChild(nodes[i]);
      }
      removeComments(nodes[i]);
    }
  }
  
  function Inliner(url, options, callback) {
    var root = url,
        inliner = this;
  
    this.requestCache = {};
    this.requestCachePending = {};
    
    // inherit EventEmitter so that we can send events with progress
    events.EventEmitter.call(this);
  
    if (typeof options == 'function') {
      callback = options;
      options = Inliner.defaults();
    } else if (options === undefined) {
      options = Inliner.defaults();
    }
    
    inliner.options = options;
    
    inliner.total = 1;
    inliner.todo = 1;
    inliner.emit('jobs', (inliner.total - inliner.todo) + '/' + inliner.total);
  
    inliner.on('error', function (data) {
      console.error(data + ' :: ' + url);
    });
    
    inliner.get(url, function (html) {
      inliner.todo--;
      inliner.emit('jobs', (inliner.total - inliner.todo) + '/' + inliner.total);
      
      // workaround for https://github.com/tmpvar/jsdom/issues/172
      // need to remove empty script tags as it casues jsdom to skip the env callback
      html = html.replace(/<\script(:? type=['"|].*?['"|])><\/script>/ig, '');
  
      if (!html) {
        inliner.emit('end', '');
        callback && callback('');
      } else {
        // BIG ASS PROTECTIVE TRY/CATCH - mostly because of this: https://github.com/tmpvar/jsdom/issues/319
        try { 
  
        jsdom.env(html, '', [
          // NOTE: Changing this for Upsight. Our legacy usage of inliner
          // involves running a local HTTP server and making requests against
          // that, so we can do things offline. This likely won't be helpful
          // for anyone else.
          'http://127.0.0.1:8080/node_modules/inliner/jquery.min.js'
        ], function(errors, window) {
          // remove jQuery that was included with jsdom
          window.$('script:last').remove();
          
          var todo = { scripts: true, images: inliner.options.images, links: true, styles: true },
              assets = {
                scripts: window.$('script'),
                images: window.$('img').filter(function(){ return this.src.indexOf('data:') == -1; }),
                links: window.$('link[rel=stylesheet]'),
                styles: window.$('style')
              },
              breakdown = {},
              images = {};
            
          inliner.total = 1;
  
          for (var key in todo) {
            if (todo[key] === true && assets[key]) {
              breakdown[key] = assets[key].length;
              inliner.total += assets[key].length;
              inliner.todo += assets[key].length;
            } else {
              assets[key] = [];
            }
          }
  
          inliner.emit('jobs', (inliner.total - inliner.todo) + '/' + inliner.total);
  
          function finished() {
            var items = 0,
                html = '';
            for (var key in breakdown) {
              items += breakdown[key];
            }
            inliner.emit('jobs', (inliner.total - inliner.todo) + '/' + inliner.total);
  
            if (items === 0) {
              // manually remove the comments
              var els = removeComments(window.document.documentElement);
              // collapse the white space
              if (inliner.options.collapseWhitespace) {
                // TODO put white space helper back in
                window.$('pre').html(function (i, html) {
                  return html.replace(/\n/g, '~~nl~~'); //.replace(/\s/g, '~~s~~');
                });
                window.$('textarea').val(function (i, v) {
                  return v.replace(/\n/g, '~~nl~~').replace(/\s/g, '~~s~~');
                });
                html = window.document.innerHTML;
                html = html.replace(/\s+/g, ' ').replace(/~~nl~~/g, '\n').replace(/~~s~~/g, ' ');
              } else {
                html = window.document.innerHTML;
              }
  
              html = '<!DOCTYPE html>' + html;
              callback && callback(html);
              inliner.emit('end', html);
            } else if (items < 0) {
              console.log('something went wrong on finish');
              console.dir(breakdown);
            } 
          }
  
          todo.images && assets.images.each(function () {
            var img = this,
                resolvedURL = URL.resolve(url, img.src);
            inliner.get(resolvedURL, { encode: true }, function (dataurl) {
              if (dataurl) images[img.src] = dataurl;
              img.src = dataurl;
              breakdown.images--;
              inliner.todo--;
              finished();
            });
          });
      
          todo.styles && assets.styles.each(function () {
            var style = this;
            inliner.getImportCSS(root, this.innerHTML, function (css, url) {
              inliner.getImagesFromCSS(url, css, function (css) {
                if (inliner.options.compressCSS) inliner.emit('progress', 'compress inline css');
                window.$(style).text(css);
  
                breakdown.styles--;
                inliner.todo--;
                finished();
              });
            });
          });
  
          todo.links && assets.links.each(function () {
            var link = this,
                linkURL = URL.resolve(url, link.href);
  
            inliner.get(linkURL, function (css) {
              inliner.getImagesFromCSS(linkURL, css, function (css) {
                inliner.getImportCSS(linkURL, css, function (css) {
                  if (inliner.options.compressCSS) inliner.emit('progress', 'compress ' + linkURL);
                  breakdown.links--;
                  inliner.todo--;
  
                  var style = '',
                      media = link.getAttribute('media');
              
                  if (media) {
                    style = '<style>@media ' + media + '{' + css + '}</style>';
                  } else {
                    style = '<style>' + css + '</style>';
                  }
  
                  window.$(link).replaceWith(style);
                  finished();
                });
              });
            });
          });
  
          function scriptsFinished() {
            inliner.emit('jobs', (inliner.total - inliner.todo) + '/' + inliner.total);
            if (breakdown.scripts == 0) {
              // now compress the source JavaScript
              assets.scripts.each(function () {
                if (this.innerHTML.trim().length == 0) {
                  // this is an empty script, so throw it away
                  inliner.todo--;
                  inliner.emit('jobs', (inliner.total - inliner.todo) + '/' + inliner.total);
                  return;
                }
  
                var $script = window.$(this),
                    src = $script.attr('src'),
                    // note: not using .innerHTML as this coerses & => &amp;
                    orig_code = this.firstChild.nodeValue
                                    .replace(/<\/script>/gi, '<\\/script>'),
                    final_code = '';
  
                // only remove the src if we have a script body
                if (orig_code) { 
                  $script.removeAttr('src');
                }
  
                // don't compress already minified code
                if(!(/\bmin\b/).test(src) && !(/google-analytics/).test(src)) { 
                  inliner.todo++;
                  inliner.total++;
                  inliner.emit('jobs', (inliner.total - inliner.todo) + '/' + inliner.total);
                  try {
                    var ast = jsp.parse(orig_code); // parse code and get the initial AST
  
                    ast = pro.ast_mangle(ast); // get a new AST with mangled names
                    ast = pro.ast_squeeze(ast); // get an AST with compression optimizations
                    final_code = pro.gen_code(ast);
  
                    // some protection against putting script tags in the body
                    window.$(this).text(final_code).append('\n');
  
                    if (src) {
                      inliner.emit('progress', 'compress ' + URL.resolve(root, src));
                    } else {
                      inliner.emit('progress', 'compress inline script');
                    }
                  } catch (e) {
                    console.error('exception on ', src);
                    console.error('exception in ' + src + ': ' + e.message);
                    console.error('>>>>>> ' + orig_code.split('\n')[e.line - 1]);
                  }
                  inliner.todo--;
                  inliner.emit('jobs', (inliner.total - inliner.todo) + '/' + inliner.total);
                } else if (orig_code) {
                  window.$(this).text(orig_code);
                  // this.innerText = orig_code;
                }
              });
              finished();
            }
          }
  
          // basically this is the jQuery instance we tacked on to the request,
          // but we're just being extra sure before we do zap it out  
          todo.scripts && assets.scripts.each(function () {
            var $script = window.$(this),
                scriptURL = URL.resolve(url, this.src);
  
            if (!this.src || scriptURL.indexOf('google-analytics.com') !== -1) { // ignore google
              breakdown.scripts--;
              inliner.todo--;
              scriptsFinished();
            } else if (this.src) {
              inliner.get(scriptURL, { not: 'text/html' }, function (data) {
                // catches an exception that was being thrown, but script escaping wasn't being caught
                if (data) $script.text(data.replace(/<\/script>/gi, '<\\/script>')); //.replace(/\/\/.*$\n/g, ''));
                // $script.before('<!-- ' + scriptURL + ' -->');
                breakdown.scripts--;
                inliner.todo--;
                scriptsFinished();
              });      
            }
          });
  
          // edge case - if there's no images, nor scripts, nor links - we call finished manually
          if (assets.links.length == 0 && 
              assets.styles.length == 0 && 
              assets.images.length == 0 && 
              assets.scripts.length == 0) {
            finished();
          }
          
          /** Inliner jobs:
           *  1. get all inline images and base64 encode
           *  2. get all external style sheets and move to inline
           *  3. get all image references in CSS and base64 encode and replace urls
           *  4. get all external scripts and move to inline
           *  5. compress JavaScript
           *  6. compress CSS & support media queries
           *  7. compress HTML (/>\s+</g, '> <');
           * 
           *  FUTURE ITEMS:
           *  - support for @import
           *  - javascript validation - i.e. not throwing errors
           */
        });
  
        } catch (e) {
          inliner.emit('error', 'Fatal error parsing HTML - exiting');
        }
      }
    });
  }
  
  util.inherits(Inliner, events.EventEmitter);
  
  Inliner.version = Inliner.prototype.version = JSON.parse(require('fs').readFileSync(__dirname + '/package.json').toString()).version;
  
  Inliner.prototype.get = function (url, options, callback) {
    // support no options being passed in
    if (typeof options == 'function') {
      callback = options;
      options = {};
    }
    
    // if we've cached this request in the past, just return the cached content
    if (this.requestCache[url] !== undefined) {
      this.emit('progress', 'cached ' + url);
      return callback && callback(this.requestCache[url]);
    } else if (this.requestCachePending[url] !== undefined) {
      this.requestCachePending[url].push(callback);
      return true;
    } else {
      this.requestCachePending[url] = [callback];
    }
  
    var inliner = this;
  
    // TODO remove the sync
    if (path.existsSync(url)) {
      // then we're dealing with a file
      fs.readFile(url, 'utf8', function (err, body) {
        inliner.requestCache[url] = body;
        inliner.requestCachePending[url].forEach(function (callback, i) {
          if (i == 0 && body) {
            inliner.emit('progress', (options.encode ? 'encode' : 'get') + ' ' + url);
          } else if (body) {
            inliner.emit('progress', 'cached ' + url);
          }
          callback && callback(body);
        });
      });
      
      return;
    }
    
    // otherwis continue and create a new web request
    var request = makeRequest(url),
        body = '';
  
    // this tends to occur when we can't connect to the url - i.e. target is down
    // note that the main inliner app handles sending the error back to the client
    request.on('error', function (error) {
      console.error(error.message, url);
      callback && callback('');
    });
  
  
    request.on('response', function (res) {
      var gunzip;
  
      // if we get a gzip header, then first we attempt to load the node-compress library
      // ...which annoyingly isn't supported anymore (maybe I should fork it...).
      // once loaded, we set up event listeners to handle data coming in and do a little
      // dance with the response object - which I'll explain... ==>
      if (res.headers['content-encoding'] == 'gzip') {
        console.error('loading gzip library');
        if (compress === null) {
          try {
            compress = require('./node-compress/lib/compress/');
          } catch (e) {
            console.error(url + ' sent gzipped header\nFailed to load node-compress - see http://github.com/remy/inliner for install directions. \nexiting');
            process.exit();
          }
        }
        gunzip = new compress.GunzipStream();
        
        // the data event is triggered by writing to the gunzip object when the response
        // receives data (yeah, further down).
        // once all the data has finished (triggerd by the response end event), we undefine
        // the gunzip object and re-trigger the response end event. By this point, we've
        // decompressed the gzipped content, and it's human readable again.
        gunzip.on('data', function (chunk) {
          body += chunk;
        }).on('end', function () {
          gunzip = undefined;
          res.emit('end');
        });
      }
      
      res.on('data', function (chunk) {
        // only process data if we have a 200 ok
        if (res.statusCode == 200) {
          if (gunzip) {
            gunzip.write(chunk);
          } else {
            body += chunk;
          }
        }
      });
      
      // required if we're going to base64 encode the content later on
      if (options.encode) {
        res.setEncoding('binary');
      }
  
      res.on('end', function () { 
        if (gunzip) {
          gunzip.end();
          return;
        } 
  
        if (res.statusCode !== 200) {
          inliner.emit('progress', 'get ' + res.statusCode + ' on ' + url);
          body = ''; // ?
        } else if (res.headers['location']) {
          return inliner.get(res.headers['location'], options, callback);
        } else {
          if (options && options.not) {
            if (res.headers['content-type'].indexOf(options.not) !== -1) {
              body = '';
            }
          }
          
          if (options.encode && res.statusCode == 200) {
            body = 'data:' + res.headers['content-type'] + ';base64,' + new Buffer(body, 'binary').toString('base64');
          }
          
          inliner.requestCache[url] = body;
          inliner.requestCachePending[url].forEach(function (callback, i) {
            if (i == 0 && body && res.statusCode == 200) {
              inliner.emit('progress', (options.encode ? 'encode' : 'get') + ' ' + url);
            } else if (body) {
              inliner.emit('progress', 'cached ' + url);
            }
            callback && callback(body);
          });
        }
      });
    }).end();
  };
  
  Inliner.prototype.getImagesFromCSS = function (rooturl, rawCSS, callback) {
    if (this.options.images === false) {
      callback && callback(rawCSS);
      return;
    }
    
    var inliner = this,
        images = {},
        urlMatch = /url\((?:['"]*)(?!['"]*data:)(.*?)(?:['"]*)\)/g,
        singleURLMatch = /url\(\s*(?:['"]*)(?!['"]*data:)(.*?)(?:['"]*)\s*\)/,
        matches = rawCSS.match(urlMatch),
        imageCount = matches === null ? 0 : matches.length; // TODO check!
    
    inliner.total += imageCount;
    inliner.todo += imageCount;
    
    function checkFinished() {
      inliner.emit('jobs', (inliner.total - inliner.todo) + '/' + inliner.total);
      if (imageCount < 0) {
        console.log('something went wrong :-S');
      } else if (imageCount == 0) {
        callback(rawCSS.replace(urlMatch, function (m, url) {
          return 'url(' + images[url] + ')';
        }));
      }
    }
    
    if (imageCount) {
      matches.forEach(function (url) {
        url = url.match(singleURLMatch)[1];
        var resolvedURL = URL.resolve(rooturl, url);
        if (images[url] === undefined) {
          inliner.get(resolvedURL, { encode: true }, function (dataurl) {
            imageCount--;
            inliner.todo--;
            if (images[url] === undefined) images[url] = dataurl;
            
            checkFinished();
          });
        } else {
          imageCount--;
          inliner.todo--;
          checkFinished();
        }
      });
    } else {
      callback(rawCSS);
    }
  };
  
  Inliner.prototype.getImportCSS = function (rooturl, css, callback) {
    // if (typeof css == 'function') {
    //   callback = css;
    //   rooturl = '';
    // }
    
    var position = css.indexOf('@import'),
        inliner = this;
  
    if (position !== -1) {
      var match = css.match(/@import\s*(.*)/);
      
      if (match !== null && match.length) {
        var url = match[1].replace(/url/, '').replace(/['}"]/g, '').replace(/;/, '').trim().split(' '); // clean up
        // if url has a length > 1, then we have media types to target
        var resolvedURL = URL.resolve(rooturl, url[0]);
        inliner.get(resolvedURL, function (importedCSS) {
          inliner.emit('progress', 'import ' + resolvedURL);
          if (url.length > 1) {
            url.shift();
            importedCSS = '@media ' + url.join(' ') + '{' + importedCSS + '}';
          }
          
          css = css.replace(match[0], importedCSS);
          inliner.getImportCSS(rooturl, css, callback);
        });          
      }
    } else {
      if (inliner.options.compressCSS) css = compressCSS(css);
      callback(css, rooturl);
    }
  };
  
  Inliner.defaults = function () { return { compressCSS: true, collapseWhitespace: true, images: true }; };
  
  var makeRequest = Inliner.makeRequest = function (url, extraOptions) {
    var oURL = URL.parse(url),
        options = {
          host: oURL.hostname,
          port: oURL.port === undefined ? (oURL.protocol+'').indexOf('https') === 0 ? 443 : 80 : oURL.port,
          path: (oURL.pathname || '/') + (oURL.search || ''), // note 0.5.0pre doesn't fill pathname if missing
          method: 'GET'
        };
  
    for (var key in extraOptions) {
      options[key] = extraOptions[key];
    }
  
    return http[oURL.protocol.slice(0, -1) || 'http'].request(options);
  };
  
  module.exports = Inliner;
  
  if (!module.parent) {
    // if this module isn't being included in a larger app, defer to the 
    // bin/inliner for the help options
    require('./bin/inliner');
  }
  