Crawl multiple pages at once This an update to my last question.

I want to process multiple pages at once pulling URLs from tier_list in the crawl_web method. The problem I am having is crawl_web already has two threads to crawl and index simultaneously, and the .join() seems to prevent opening more than one page.

import urllib2, collections, threading, robotparser, urlparse, Queue
from bs4 import BeautifulSoup
q = Queue.Queue()

def get_page_links(page, crawled, to_crawl, next_tier, robot_set):
    #respect the robots.txt and returns all urls from page
    rp = robotparser.RobotFileParser()
    links = []    
    uparse = urlparse.urlparse
    for link in page[1].find_all('a'):
        url = link.get('href')
        base_url = (uparse(page[0]).scheme + '://' + uparse(page[0]).netloc)
        if base_url not in robot_set:
            robot_set.add(base_url)
            rp.set_url(urlparse.urljoin(base_url,'robots.txt'))
            rp.read()
        abs_url = urlparse.urljoin(base_url, url)
        if rp.can_fetch('*', abs_url): 
            links.append(abs_url)
    links = set(links)
    next_tier.extend(links)
    to_crawl.update(links)
    to_crawl.remove(page[0])
    crawled.add(page[0])

def get_page(url):
    #return page html from url
    try:
        return [url,BeautifulSoup(urllib2.urlopen(url).read())]
    except:
        return [url,BeautifulSoup("")] 

def get_page_words(page, word_index):
    #pulls words off page to be indexed
    url = page[0]
    words = page[1].get_text()    
    to_remove = '{}[]«#$%^&amp;*._,1234567890+=&lt;&gt;/\()":;!?'
    bad_words = set(['http','www','com','https','gov','org','edu','imgs'])
    for i in to_remove:
        words = words.replace(i, ' ').lower()       
    words = words.split()
    for word in words:
        if word in word_index:
            word_index[word].add(url)
        elif len(word) &lt; 15 and word not in bad_words:
            word_index[word] = set([url])

def crawl_web(url, maxdepth):
    #controls crawl depth, updates url lists and word_index 
    depth = 0
    robot_set = set()
    crawled = set()
    to_crawl = set([url])
    word_index = {}
    tier_list = [[url]]
    while depth &lt; maxdepth:
        next_tier = []
        for url in tier_list[depth]:
            page = get_page(url)
            t1 = threading.Thread(target = get_page_links, args = (page, 
                                      crawled, to_crawl, next_tier, robot_set))
            t2 = threading.Thread(target = get_page_words, args = (page,
                                                                   word_index))
            t1.start()
            t2.start()
            t1.join()
            t2.join()
        tier_list.append(next_tier)
        depth += 1
    return crawled, to_crawl, word_index, tier_list

def search_engine(target_string, word_index):
    #searches word_bank for words in string, returns urls words are found at
    targets = list(set(target_string.split()))
    result =[]
    for word in targets:
        if word in word_index:
            result += word_index[word]
    ans = collections.Counter(result).most_common(3)
    return ans

result = crawl_web('http://xkcd.com',2)
word_index = result[2]
print result[0]
print search_engine('starting blogs about', word_index)
While I can't answer to your main question about crawling multiple pages at once,
I have a couple of comments about this code.

get_page_links

In this code, it seems that base_url and the if that follows should be outside of the for loop:


links = []
for link in page[1].find_all('a'):
    url = link.get('href')
    base_url = (uparse(page[0]).scheme + '://' + uparse(page[0]).netloc)
    if base_url not in robot_set:
        robot_set.add(base_url)
        rp.set_url(urlparse.urljoin(base_url,'robots.txt'))
        rp.read()
    abs_url = urlparse.urljoin(base_url, url)
    if rp.can_fetch('*', abs_url): 
        links.append(abs_url)
links = set(links)



Also, instead of links = set(links),
it would be better to make links a set in the first place.

Rewriting the above code like this would be more efficient:

links = set()
base_url = uparse(page[0]).scheme + '://' + uparse(page[0]).netloc
if base_url not in robot_set:
    robot_set.add(base_url)
    rp.set_url(urlparse.urljoin(base_url, 'robots.txt'))
    rp.read()
for link in page[1].find_all('a'):
    url = link.get('href')
    abs_url = urlparse.urljoin(base_url, url)
    if rp.can_fetch('*', abs_url):
        links.add(abs_url)


get_page

It would be better to specify in the except clause the type of error that can be raised.

get_page_words

This expression contains the non-ascii character «,
and the posted code doesn't contain an encoding declaration:


to_remove = '{}[]«#$%^&amp;*._,1234567890+=&lt;&gt;/\()":;!?'



You can declare utf-8 encoding by adding this line at the top of the file:

# coding=utf-8





to_remove = '{}[]«#$%^&amp;*._,1234567890+=&lt;&gt;/\()":;!?'
for i in to_remove:
    words = words.replace(i, ' ').lower()



One problem here is that words is lowercased in every iteration,
when it would be enough to lowercase just once, before starting the loop.

But the bigger problem is the loop itself:
it would be much more efficient to use a regular expression to get rid of all the unwanted characters, for example like this:

import re
re_nonword = re.compile(r'[{}[\]«#$%^&amp;*._,1234567890+=&lt;&gt;/\()":;!?]')
words = re_nonword.sub(' ', words)


set literals

Instead of this:


bad_words = set(['http','www','com','https','gov','org','edu','imgs'])



As of Python 2.7, you can write using set literals, like this:

bad_words = {'http', 'www', 'com', 'https', 'gov', 'org', 'edu', 'imgs'}


I also put spaces after the commas, to follow PEP8.