func NewScraper(u *url.URL, content string) (*Scraper, error) {
    var title string
    var language string
    var author string
    var description string
    var generator string
    var feed string
    var metaImage string
    charset := "utf-8"
    links := make([]string, 0)
    images := make([]string, 0)
    keywords := make([]string, 0)
    compatibility := make(map[string]string)

    scrpr := func(n *html.Node) {
        switch n.Data {
        case "html":
            language = findAttribute(n, "lang")
        case "title":
            title = n.FirstChild.Data
        case "a":
            links = addElement(links, u, n, "href")
        case "img":
            images = addElement(images, u, n, "src")
        case "link":
            typ := findAttribute(n, "type")
            switch typ {
            case "application/rss+xml":
                feed = findAttribute(n, "href")
            }
        case "meta":
            name := findAttribute(n, "name")
            switch name {
            case "author":
                author = findAttribute(n, "content")
            case "keywords":
                keywords = strings.Split(findAttribute(n, "content"), ", ")
            case "description":
                description = findAttribute(n, "content")
            case "generator":
                generator = findAttribute(n, "content")
            case "image":
                metaImage = findAttribute(n,"content")
            }

            httpEquiv := findAttribute(n, "http-equiv")
            switch httpEquiv {
            case "Content-Type":
                charset = findCharset(findAttribute(n, "content"))
            case "X-UA-Compatible":
                compatibility = mapifyStr(findAttribute(n, "content"))
            }
        }
    }
    save := &CrawlerSave{}
    save.Init()
    save.Url = u.String()
    tree, err := save.Parse(content)
    if err != nil {
        return nil, err
    }
    tree.Walk(scrpr)

    return &Scraper{title,
        language,
        author,
        description,
        metaImage,
        generator,
        feed,
        charset,
        links,
        images,
        keywords,
        compatibility}, nil
}
// newScraper parses the given url and scrapes the site, returning
// a scraper object with all the site info and meta tags
newScraper parses the given url and scrapes the site, returning
a scraper object with all the site info and meta tags
// newScraper parses the given url and scrapes the site returning // a scraper object with all the site info and meta tags
func NewScraper(u *url.URL, content string) (*Scraper, error) {
    var title string
    var language string
    var author string
    var description string
    var generator string
    var feed string
    var metaImage string
    charset := "utf-8"
    links := make([]string, 0)
    images := make([]string, 0)
    keywords := make([]string, 0)
    compatibility := make(map[string]string)
    scrpr := func(n *html.Node) {
        switch n.Data {
        case "html":
            language = findAttribute(n, "lang")
        case "title":
            title = n.FirstChild.Data
        case "a":
            links = addElement(links, u, n, "href")
        case "img":
            images = addElement(images, u, n, "src")
        case "link":
            typ := findAttribute(n, "type")
            switch typ {
            case "application/rss+xml":
                feed = findAttribute(n, "href")
            }
        case "meta":
            name := findAttribute(n, "name")
            switch name {
            case "author":
                author = findAttribute(n, "content")
            case "keywords":
                keywords = strings.Split(findAttribute(n, "content"), ", ")
            case "description":
                description = findAttribute(n, "content")
            case "generator":
                generator = findAttribute(n, "content")
            case "image":
                metaImage = findAttribute(n,"content")
            }
            httpEquiv := findAttribute(n, "http-equiv")
            switch httpEquiv {
            case "Content-Type":
                charset = findCharset(findAttribute(n, "content"))
            case "X-UA-Compatible":
                compatibility = mapifyStr(findAttribute(n, "content"))
            }
        }
    }
    save := &CrawlerSave{}
    save.Init()
    save.Url = u.String()
    tree, err := save.Parse(content)
    if err != nil {
        return nil, err
    }
    tree.Walk(scrpr)
    return &Scraper{title,
        language,
        author,
        description,
        metaImage,
        generator,
        feed,
        charset,
        links,
        images,
        keywords,
        compatibility}, nil
}
func NewScraper ( u * url . URL , content string ) ( * Scraper , error ) { var title string var language string var author string var description string var generator string var feed string var metaImage string charset := " " links := make ( [ ] string , 0 ) images := make ( [ ] string , 0 ) keywords := make ( [ ] string , 0 ) compatibility := make ( map [ string ] string ) scrpr := func ( n * html . Node ) { switch n . Data { case " " : language = findAttribute ( n , " " ) case " " : title = n . FirstChild . Data case " " : links = addElement ( links , u , n , " " ) case " " : images = addElement ( images , u , n , " " ) case " " : typ := findAttribute ( n , " " ) switch typ { case " " : feed = findAttribute ( n , " " ) } case " " : name := findAttribute ( n , " " ) switch name { case " " : author = findAttribute ( n , " " ) case " " : keywords = strings . Split ( findAttribute ( n , " " ) , " " ) case " " : description = findAttribute ( n , " " ) case " " : generator = findAttribute ( n , " " ) case " " : metaImage = findAttribute ( n , " " ) } httpEquiv := findAttribute ( n , " " ) switch httpEquiv { case " " : charset = findCharset ( findAttribute ( n , " " ) ) case " " : compatibility = mapifyStr ( findAttribute ( n , " " ) ) } } } save := & CrawlerSave { } save . Init ( ) save . Url = u . String ( ) tree , err := save . Parse ( content ) if err != nil { return nil , err } tree . Walk ( scrpr ) return & Scraper { title , language , author , description , metaImage , generator , feed , charset , links , images , keywords , compatibility } , nil }
newScraper parses the given url and scrapes the site, returning
a scraper object with all the site info and meta tags
newScraper parses the given url and scrapes the site returning a scraper object with all the site info and meta tags