How to crawl latest articles in a specific site using specific set keyword?
Question: I am trying a python code for crawling article links on specific sites based
on key word like name of the article.but i didn't get the links appropriate
links.

    
    
    import sys
    import requests
    from bs4 import BeautifulSoup
    import urllib.request
    from urlparse import urlparse
    def extract_article_links(url,data):
        req = urllib.request.Request(url,data)
        response = urllib.request.urlopen(req)
        responseData = response.read()
        #r = requests.get(url)
        soup = BeautifulSoup(responseData.content)
        links = soup.find_all('a')
        for link in links:
           try:
                #if 'http' in link:
                print ("<a href='%s'>%s</a>" % (link.get('href'),link.text))
           except Exception as e :
                print (e)
        responseData = soup.find_all("div",{"class:info"})
        print responseData
        for item in responseData:
            print (item.contents[0].text)
            print (item.contents[1].text)
        if __name__ == "__main__":
           from sys import argv
           if (len(argv)<2):
              print"Insufficient arguments..!!"
              sys.exit(1)
            url = sys.argv[1]
            values = {'s':'article','submit':'search'}
            data = urlparse.urlencode(values)
            data = data.encode('utf-8')
            extract_article_links(url,data)
    


Answer: Try [lxml](http://lxml.de/), analyze the html and locate elements you are
looking for, then you can do this easily with xpath :

    
    
    from lxml import html
    print map (lambda link: link, html.fromstring(source).xpath('//a/@href'))
    

of course you need to modify the xpath according to the attribute you are
looking for.

