#!/usr/bin/env python
# Luke Matarazzo
# Pen Testing - Email Scraper

import urllib2
from urlparse import urlparse
import re
import argparse
from BeautifulSoup import BeautifulSoup, SoupStrainer
from sys import exit

#create the email regex object once for efficiency
emailRegex = re.compile("[\w.%+-]+@[\w.-]+\.[a-zA-Z]+")

def validateLink(baseUrl, newUrl):
	"""
	Make sure a given link is a full, valid URL and is in our domain.

	Purpose: Take in a relative URL or path name and prepend the correct domain name to it.
			 And if it is a full URL but in a different domain, make it blank.
	Parameters: baseUrl - the original, parent domain/link
				newUrl - the relative pathname or URL
	Returns: Returns the new URL in the correct format. Blank if not in the same domain.
	"""
	if newUrl[:1] == "/": #if a relative URL, add domain to beginning
		newUrl = baseUrl + newUrl
	else: #if full link, make sure it's in the same domain
		baseDomain = '{uri.netloc}'.format(uri=urlparse(baseUrl))
		newDomain = '{uri.netloc}'.format(uri=urlparse(newUrl))
		
		#if not in same domain, make it a blank string
		if baseDomain != newDomain:
			newUrl = ""

	return newUrl

def scrape(url, getLinks="yes"):
	"""
	Scrape a web page for links and emails.

	Purpose: Download a given URL's HTML and parse it for links and emails. All links added to the
			 list will be in the same domain as the given url.
	Parameters: url - the URL that is going to be scraped
	Returns: A dictionary containing two separate lists that consist of links and emails.
	"""
	#visit URL given
	try:
		opener = urllib2.build_opener()
		opener.addheaders = [("User-agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; it; "
							  "rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11")]
		result = opener.open(url).read()
	except: #if error, raise except, we'll catch it later
		raise

	#if we get a blank page
	if not result:
		print "It seems there was nothing at the given URL."
		return {"links": [], "emails": []}

	scrapeLinks = []
	scrapeEmails = []

	#get all the links on the page
	if getLinks == "yes":
		for link in BeautifulSoup(result, parseOnlyThese=SoupStrainer("a")):
			if link.has_key("href"):
				#make sure link is in domain and a full valid link
				linkRef = validateLink(url, str(link["href"]))

				#check to make sure link is not empty and we don't already have it in the list
				if linkRef and linkRef not in scrapeLinks:
					scrapeLinks.append(linkRef)

	scrapeEmails = list(set(emailRegex.findall(result))) #get all unique emails

	return {"links": scrapeLinks, "emails": scrapeEmails} #return as a dictionary

def main():
	#parser stuff
	parser = argparse.ArgumentParser()
	parser.add_argument("url", help="This is the URL that is to be scraped")
	parser.add_argument("-n", "--number-of-emails", "--emails", type=int, dest="numEmails",
						default=20, help="Number of emails before the scraper stops")
	parser.add_argument("-f", "--file", default="emails.txt", dest="outfile",
						help="Output file that will hold the emails")
	parser.add_argument("-d", "--delimiter", dest="delim", default="\n",
						help="Delimiter for the output email file. Default is a newline")
	verbosityGroup = parser.add_mutually_exclusive_group()
	verbosityGroup.add_argument("-q", "--quiet", action="store_true",
						help="Enable quiet mode for no standard output")
	verbosityGroup.add_argument("-v", "--verbose", action="store_true",
								help="Enable verbose mode")
	args = parser.parse_args()

	#make sure it starts with http:// or https://, otherwise add http://
	if args.url[:7] != "http://" and args.url[:8] != "https://":
		args.url = "http://" + args.url

	if not args.quiet:
		print "Scraping started..."

	#get the urls and emails from the first URL
	try:
		firstPage = scrape(args.url)
	except Exception, e:
		print "***", e, "-", args.url, "***"
		return -1

	if args.verbose:
		print "'%s' was scraped successfully for links and emails" % args.url
		print "Number of links found %d" % len(firstPage["links"]), "\n"

	#loop through all given links and get emails on those pages
	for link in firstPage["links"]:
		if len(firstPage["emails"]) >= args.numEmails: #if too many emails, we're done
			if args.verbose:
				print "Max number of emails hit"
			break

		if args.verbose:
			print "Scraping '%s' ..." % link,

		#try to get emails, print error if there is one
		try:
			tempPage = scrape(link, getLinks="no")
		except Exception, e:
			if not args.quiet:
				print "***", e, "***"
			continue

		if args.verbose:
			print "scraped successfully for emails"

		#loop through emails found, add if new and not over max number of emails
		for newEmail in tempPage["emails"]:
			if len(firstPage["emails"]) >= args.numEmails: #if too many emails, we're done
				if args.verbose:
					print "Max number of emails hit"
				break

			if newEmail not in firstPage["emails"]: #if unique, add
				firstPage["emails"].append(newEmail)

	#write emails to file based on delimiter
	if firstPage["emails"]:
		with open(args.outfile, "w") as f:
			for line in firstPage["emails"]:
				f.write(line + args.delim)

			if args.delim != "\n": #add a final newline if necessary
				f.write("\n")

		if not args.quiet:
			print "Emails written to", args.outfile
	else:
		if not args.quiet:
			print "No emails found"

if __name__ == "__main__":
	main()
