lib/web_page_archiver.rb
def convert(filename_or_uri)
      base_file = ""
      begin
        base_file = Typhoeus.get(filename_or_uri)
      rescue => ex
        print "!H".colorize(:red)
        print " \"#{ex}\" ".gsub("\n", " ").gsub("\r", " ").colorize(:yellow)
        sleep(3.seconds)
        base_file = Typhoeus.get(filename_or_uri)
      end

      @parser = Nokogiri::HTML(base_file.body.to_s)

      @parser.search('img').each { |i|
        uri = i.attr('src')
        uri = URI.encode(uri)
        uri = join_uri( filename_or_uri, uri).to_s
        uid = Digest::MD5.hexdigest(uri)
        @contents[uid] = {:uri=>uri, :parser_ref=>i, :attribute_name=>'src'}
        i.set_attribute('src',"cid:#{uid}")
      }

      @parser.search('link[rel=stylesheet]').each { |i|
        uri = i.attr('href')
        uri = URI.encode(uri)
        uri = join_uri( filename_or_uri, uri)
        uid = Digest::MD5.hexdigest(uri)
        @contents[uid] = {:uri=>uri, :parser_ref=>i, :attribute_name=>'href'}
        i.set_attribute('href',"cid:#{uid}")
      }

      @parser.search('script').each { |i|
        next unless i.attr('src')
        uri = i.attr('src')
        uri = URI.encode(uri)
        uri = join_uri( filename_or_uri, uri)
        uid = Digest::MD5.hexdigest(uri)
        @contents[uid] = {:uri=>uri, :parser_ref=>i, :attribute_name=>'src'}
        i.set_attribute('src',"cid:#{uid}")
      }

      self.set_contents
      return @parser.to_s
    end
# convert object at uri to self-contained text-file
#
# @param [String, URI] filename_or_uri to test for
# @return [String] text blob containing the result
#

convert object at uri to self-contained text-file
convert object at uri to self - contained text - file
def convert(filename_or_uri)
      base_file = ""
      begin
        base_file = Typhoeus.get(filename_or_uri)
      rescue => ex
        print "!H".colorize(:red)
        print " \"#{ex}\" ".gsub("\n", " ").gsub("\r", " ").colorize(:yellow)
        sleep(3.seconds)
        base_file = Typhoeus.get(filename_or_uri)
      end
      @parser = Nokogiri::HTML(base_file.body.to_s)
      @parser.search('img').each { |i|
        uri = i.attr('src')
        uri = URI.encode(uri)
        uri = join_uri( filename_or_uri, uri).to_s
        uid = Digest::MD5.hexdigest(uri)
        @contents[uid] = {:uri=>uri, :parser_ref=>i, :attribute_name=>'src'}
        i.set_attribute('src',"cid:#{uid}")
      }
      @parser.search('link[rel=stylesheet]').each { |i|
        uri = i.attr('href')
        uri = URI.encode(uri)
        uri = join_uri( filename_or_uri, uri)
        uid = Digest::MD5.hexdigest(uri)
        @contents[uid] = {:uri=>uri, :parser_ref=>i, :attribute_name=>'href'}
        i.set_attribute('href',"cid:#{uid}")
      }
      @parser.search('script').each { |i|
        next unless i.attr('src')
        uri = i.attr('src')
        uri = URI.encode(uri)
        uri = join_uri( filename_or_uri, uri)
        uid = Digest::MD5.hexdigest(uri)
        @contents[uid] = {:uri=>uri, :parser_ref=>i, :attribute_name=>'src'}
        i.set_attribute('src',"cid:#{uid}")
      }
      self.set_contents
      return @parser.to_s
    end
def convert ( filename_or_uri ) base_file = "" begin base_file = Typhoeus . get ( filename_or_uri ) rescue => ex print "!H" . colorize ( :red ) print " \"#{ex}\" " . gsub ( "\n" , " " ) . gsub ( "\r" , " " ) . colorize ( :yellow ) sleep ( 3 . seconds ) base_file = Typhoeus . get ( filename_or_uri ) end @parser = Nokogiri :: HTML ( base_file . body . to_s ) @parser . search ( 'img' ) . each { | i | uri = i . attr ( 'src' ) uri = URI . encode ( uri ) uri = join_uri ( filename_or_uri , uri ) . to_s uid = Digest :: MD5 . hexdigest ( uri ) @contents [ uid ] = { :uri => uri , :parser_ref => i , :attribute_name => 'src' } i . set_attribute ( 'src' , "cid:#{uid}" ) } @parser . search ( 'link[rel=stylesheet]' ) . each { | i | uri = i . attr ( 'href' ) uri = URI . encode ( uri ) uri = join_uri ( filename_or_uri , uri ) uid = Digest :: MD5 . hexdigest ( uri ) @contents [ uid ] = { :uri => uri , :parser_ref => i , :attribute_name => 'href' } i . set_attribute ( 'href' , "cid:#{uid}" ) } @parser . search ( 'script' ) . each { | i | next unless i . attr ( 'src' ) uri = i . attr ( 'src' ) uri = URI . encode ( uri ) uri = join_uri ( filename_or_uri , uri ) uid = Digest :: MD5 . hexdigest ( uri ) @contents [ uid ] = { :uri => uri , :parser_ref => i , :attribute_name => 'src' } i . set_attribute ( 'src' , "cid:#{uid}" ) } self . set_contents return @parser . to_s end
convert object at uri to self-contained text-file
convert object at uri to self - contained text - file
text blob containing the result
text blob containing the result
String
filename_or_uri
filename_or_uri to test for
filename_or_uri to test for