main.py
def crawl2(url, depth, first_call = 0, home_tags = None):
	'''
	Crawl the yt web graph and return relevance index
	upto defined depth.
	'''
	print("---\tat depth = {}".format(depth))
	if depth == 0:
		return []
	
	tags = home_tags
	links = None
	attempt=5
	while (tags==None or links==None) and attempt>0:
		attempt -= 1
		tags, links = f.getDataFromUrl(url)
		if tags == None or links == None:
			print("---\t---\tpage issues...retrying")
			continue
	
	if attempt==0:
		print("---\t---\tran out of attempts")
		return []

	temp_tags = None
	attempt = 10
	while temp_tags==None and attempt>0:
		attempt -= 1
		link = random.choice(links)
		temp_tags, _ = f.getDataFromUrl(link)
		if temp_tags == None:
			print("---\t---\tpage issues...skipping page")
			continue
	if attempt==0:
		print("---\t---\tran out of attempts")
		return []
	relevance_list = []
	relevance = 0
	if first_call==0:
		tags = home_tags
	relevance = f.getRelevance(tags, temp_tags)
	relevance_list.append(relevance)
	return_list = crawl2(link, depth-1, 0, tags)
	relevance_list = relevance_list + return_list
	return relevance_list

	Crawl the yt web graph and return relevance index
	upto defined depth.
	
Crawl the yt web graph and return relevance index
upto defined depth.
Crawl the yt web graph and return relevance index upto defined depth .
def crawl2(url, depth, first_call = 0, home_tags = None):
	print("---\tat depth = {}".format(depth))
	if depth == 0:
		return []
	tags = home_tags
	links = None
	attempt=5
	while (tags==None or links==None) and attempt>0:
		attempt -= 1
		tags, links = f.getDataFromUrl(url)
		if tags == None or links == None:
			print("---\t---\tpage issues...retrying")
			continue
	if attempt==0:
		print("---\t---\tran out of attempts")
		return []
	temp_tags = None
	attempt = 10
	while temp_tags==None and attempt>0:
		attempt -= 1
		link = random.choice(links)
		temp_tags, _ = f.getDataFromUrl(link)
		if temp_tags == None:
			print("---\t---\tpage issues...skipping page")
			continue
	if attempt==0:
		print("---\t---\tran out of attempts")
		return []
	relevance_list = []
	relevance = 0
	if first_call==0:
		tags = home_tags
	relevance = f.getRelevance(tags, temp_tags)
	relevance_list.append(relevance)
	return_list = crawl2(link, depth-1, 0, tags)
	relevance_list = relevance_list + return_list
	return relevance_list
def crawl2 ( url , depth , first_call = 0 , home_tags = None ) : print ( "---\tat depth = {}" . format ( depth ) ) if depth == 0 : return [ ] tags = home_tags links = None attempt = 5 while ( tags == None or links == None ) and attempt > 0 : attempt -= 1 tags , links = f . getDataFromUrl ( url ) if tags == None or links == None : print ( "---\t---\tpage issues...retrying" ) continue if attempt == 0 : print ( "---\t---\tran out of attempts" ) return [ ] temp_tags = None attempt = 10 while temp_tags == None and attempt > 0 : attempt -= 1 link = random . choice ( links ) temp_tags , _ = f . getDataFromUrl ( link ) if temp_tags == None : print ( "---\t---\tpage issues...skipping page" ) continue if attempt == 0 : print ( "---\t---\tran out of attempts" ) return [ ] relevance_list = [ ] relevance = 0 if first_call == 0 : tags = home_tags relevance = f . getRelevance ( tags , temp_tags ) relevance_list . append ( relevance ) return_list = crawl2 ( link , depth - 1 , 0 , tags ) relevance_list = relevance_list + return_list return relevance_list
Crawl the yt web graph and return relevance index
upto defined depth.
Crawl the yt web graph and return relevance index upto defined depth .
'''
	Crawl the yt web graph and return relevance index
	upto defined depth.
	'''
url
depth
first_call
home_tags