text: ﻿import os
  import sys
  import urllib2
  from bs4 import BeautifulSoup, Comment
  import collections
  import math
  import csv
  import re
  import numpy as np
  import json
  import urlparse
  import inspect
  import argparse
  import traceback
  import datetime
  import time
  import codecs
  
  import Main_Extractor as main_ext
  
  from operator import itemgetter
  from selenium import webdriver
  from selenium.webdriver.support.ui import WebDriverWait
  from selenium.webdriver.support import expected_conditions as EC
  from selenium.webdriver.common.by import By
  from selenium.common.exceptions import TimeoutException
  from selenium.webdriver.firefox.options import Options
  
  from common import shared
  from common import bsoup
  from common import services
  #from common import page_group
  from common import recurse_ftp as rec_ftp
  from common import spreadsheet as sh
  
  class PT_Extractor(main_ext.Extractor):
  	def __init__(self):
  		''' Initializer for the Extractor class. '''
  
  		# Set up the initial parameters
  		self.province = 'Manitoba'
  		
  		# Initialize the Main Extractor to use its variables
  		main_ext.Extractor.__init__(self)
  		
  		# Create the page groups dictionary
  		self.page_groups = []
  
  		# Create the list for the page IDs of the MLI categories
  		self.mli_opts = collections.OrderedDict()
  		self.mli_opts['Administrative Boundaries'] = 'admin'
  		self.mli_opts['Base Maps'] = 'maps'
  		self.mli_opts['Environment'] = 'environ'
  		self.mli_opts['Geographical Names'] = 'names'
  		self.mli_opts['Land Use/Cover Maps'] = 'land'
  		self.mli_opts['Transportation'] = 'transport'
  		self.mli_opts['Cadastral'] = 'cadast'
  		self.mli_opts['Digital Elevation Models'] = 'dem'
  		self.mli_opts['Digital Imagery'] = 'image'
  		self.mli_opts['Forest Inventory'] = 'forest'
  		self.mli_opts['Geology Mapping'] = 'geol'
  		self.mli_opts['Hydrography'] = 'hydro'
  		self.mli_opts['Municipal Maps'] = 'munic'
  		self.mli_opts['Quarter Section Grids'] = 'grids'
  		self.mli_opts['Soil Classification'] = 'soil'
  		self.mli_opts['Spatial Referencing'] = 'spatref'
  		self.mli_opts['Topographic Maps'] = 'topos'
  		self.mli_opts['Town & Village Plans'] = 'towns'
  		
  		####################################################################
  		# Create Services page group
  		
  		srv_grp = main_ext.PageGroup('services', "Manitoba Services")
  		
  		# No arguments to add
  		
  		# Add URLs
  		srv_grp.add_url('petrol_url', 'http://maps.gov.mb.ca/arcgis/rest/services/MG_PETROLEUM_CLIENT/MapServer')
  		srv_grp.add_url('rest_url', 'https://services.arcgis.com/mMUesHYPkXjaFGfS/ArcGIS/rest/services')
  		
  		# Add to Extractor's page group list
  		self.page_groups.append(srv_grp)
  		
  		
  		####################################################################
  		# Create MLI page group
  
  		mli_grp = main_ext.PageGroup('mli', "Manitoba Land Initiative")
  		
  		# Add arguments
  		sb_arg = mli_grp.add_arg('subpage', default='all', debug=True)
  		for k, v in self.mli_opts.items():
  			sb_arg.add_opt(k, url_tags=[v])
  		
  		# Add URLs
  		mli_grp.add_url('main_url', 'http://mli2.gov.mb.ca/mli_data/index.html')
  		
  		# Add to Extractor's page group list
  		self.page_groups.append(mli_grp)
  		
  		
  		####################################################################
  		# Create Misc page group
  
  		misc_grp = main_ext.PageGroup('misc', "Miscellaneous Web Pages")
  		
  		# No arguments to add
  		
  		# Add URLs
  		misc_grp.add_url('petrol_url', 'http://www.manitoba.ca/iem/petroleum/gis/index.html')
  		misc_grp.add_url('fire_url', 'http://www.gov.mb.ca/sd/fire/Fire-Maps/')
  		misc_grp.add_url('fireidx_url', 'http://www.gov.mb.ca/sd/fire/Fire-Maps/fireview/')
  		
  		# Add to Extractor's page group list
  		self.page_groups.append(misc_grp)
  		
  		
  		####################################################################
  		# Create DEM page group
  
  		dem_grp = main_ext.PageGroup('dem', "DEMSM Downloads and Ordering")
  		
  		# No arguments to add
  		
  		# Add URLs
  		dem_grp.add_url('main_url', 'http://www.gov.mb.ca/iem/geo/demsm/downloads.html')
  		dem_grp.add_url('specs_url', 'http://www.gov.mb.ca/iem/geo/demsm/doispecs.html')
  		dem_grp.add_url('mdata_url', 'http://www.gov.mb.ca/iem/geo/demsm/metadata.html')
  		
  		# Add to Extractor's page group list
  		self.page_groups.append(dem_grp)
  		
  		
  		####################################################################
  		# Create Maps page group
  		
  		map_grp = main_ext.PageGroup('maps', "Manitoba Interactive Maps")
  		
  		# No arguments to add
  		
  		# Add URLs
  		map_grp.add_url('petrol_url', ('http://www.manitoba.ca/iem/petroleum/gis/index.html',
  								   'https://web33.gov.mb.ca/mapgallery/mgp.html',
  								   'http://maps.gov.mb.ca/arcgis/rest/services/MG_PETROLEUM_CLIENT/MapServer'))
  		map_grp.add_url('drought_url', 'http://www.gov.mb.ca/sd/waterstewardship/water_info/drought/index.html')
  		map_grp.add_url('drink_url',
  					'http://www.gov.mb.ca/sd/waterstewardship/odw/public-info/boil-water/water_advisories_in_mb.html')
  		map_grp.add_url('weather_url', ('http://www.gov.mb.ca/sd/fire/Wx-Display/weatherview/weatherview.html',
  									'http://www.gov.mb.ca/sd/fire/Wx-Display/weatherview/data/wx_last48.csv'))
  		map_grp.add_url('fireview_url', ('http://www.gov.mb.ca/sd/fire/Fire-Maps/fireview/fireview.html',
  									 'http://www.gov.mb.ca/sd/fire/Fire-Maps/'))
  		map_grp.add_url('restrict_url', 'http://www.gov.mb.ca/sd/fire/Restrictions/index.html')
  		map_grp.add_url('wild_url', ('http://www.gov.mb.ca/sd/wildlife/habcons/wmas/index.html',
  								 'http://www.gov.mb.ca/sd/wildlife/habcons/wmas/gMap/index.html'))
  		map_grp.add_url('beach_url', ('http://www.gov.mb.ca/sd/waterstewardship/quality/beaches.html',
  								  'http://www.gov.mb.ca/sd/waterstewardship/quality/beach_table.html'))
  		
  		# Add to Extractor's page group list
  		self.page_groups.append(map_grp)
  		
  		
  		####################################################################
  		# Create Municipal page group
  		
  		mun_grp = main_ext.PageGroup('municipal', "Municipal Pages")
  		
  		# Add arguments
  		mun_grp.add_arg('word', title='Search Word')
  		
  		cat_arg = mun_grp.add_arg('category', default='all', title='Winnipeg Catalogue Category')
  		cat_arg.add_opt('Assessment, Taxation, & Corporate', ['corp'], ['Assessment%2C+Taxation%2C+%26+Corporate'])
  		cat_arg.add_opt('Cemeteries', url_tags=['Cemeteries'])
  		cat_arg.add_opt('Census', url_tags=['Census'])
  		cat_arg.add_opt('City Planning', ['planning'], ['City+Planning'])
  		cat_arg.add_opt('Contact Centre - 311', ['contact'], ['Contact+Centre+-+311'])
  		cat_arg.add_opt('Council Services', ['council'], ['Council+Services'])
  		cat_arg.add_opt('Crime', url_tags=['Crime'])
  		cat_arg.add_opt('Development Approvals, Building Permits, & Inspections', ['development'], ['Development+Approvals%2C+Building+Permits%2C+%26+Inspections'])
  		cat_arg.add_opt('Education and Training', ['education'], ['Education+and+Training'])
  		cat_arg.add_opt('Fire and Rescue Response', ['fire'], ['Fire+and+Rescue+Response'])
  		cat_arg.add_opt('Insect Control', ['insect'], ['Insect+Control'])
  		cat_arg.add_opt('Libraries', url_tags=['Libraries'])
  		cat_arg.add_opt('Organizational Support Services', ['org'], ['Organizational+Support+Services'])
  		cat_arg.add_opt('Parking', url_tags=['Parking'])
  		cat_arg.add_opt('Parks', url_tags=['Parks'])
  		cat_arg.add_opt('Province of Manitoba', ['province'], ['Province+of+Manitoba'])
  		cat_arg.add_opt('Recreation', url_tags=['Recreation'])
  		cat_arg.add_opt('Tickets and Adjudication', ['tickets'], ['Tickets+and+Adjudication'])
  		cat_arg.add_opt('Transportation Planning & Traffic Management', ['traffic'], ['Transportation+Planning+%26+Traffic+Management'])
  		cat_arg.add_opt('Water and Waste', ['waste'], ['Water+and+Waste'])
  		
  		sb_arg = mun_grp.add_arg('subpage', default='all', debug=True)
  		sb_arg.add_opt('Brandon Open Data', ['br_open'], ['bdn_opendata'])
  		sb_arg.add_opt('Brandon REST Services', ['br_rest'], ['bdn_rest'])
  		sb_arg.add_opt('Brandon GIS', ['br_gis'], ['bdn_gis'])
  		sb_arg.add_opt('Winnipeg Catalogue', ['wpg_cat'], ['wpg_catalogue'])
  		sb_arg.add_opt('Winnipeg Property Map/Aerial Photography & ServiceStat', ['wpg_prop'], ['wpg_property'])
  		
  		# Add URLs
  		mun_grp.add_url('brandon_opendata_url', 'http://opengov.brandon.ca/OpenDataService/opendata.html')
  		mun_grp.add_url('brandon_rest_url', 'https://gisapp.brandon.ca/arcgis/rest/services')
  		mun_grp.add_url('brandon_gis_url', 'http://gis.brandon.ca')
  		mun_grp.add_url('winnipeg_opendata_url', 'https://data.winnipeg.ca/browse')
  		mun_grp.add_url('winnipeg_property_url', 'http://winnipeg.ca/ppd/maps_aerial.stm')
  		mun_grp.add_url('winnipeg_servstat_url', 'http://winnipeg.ca/Interhom/serviceStat')
  		
  		# Add to Extractor's page group list
  		self.page_groups.append(mun_grp)
  		
  
  	###################################################################################################################
  
  	def print_dict(self, in_dict):
  		# dict_list = ["%s: '%s'" % (k, v) for k, v in in_dict.items()]
  		dict_list = []
  		for k, v in in_dict.items():
  			try:
  				dict_list.append("%s: '%s'" % (k, v))
  				print '\t\t'.join(dict_list)
  			except Exception as e:
  				continue
  
  	def get_mdata_item(self, soup, heading, field, tag='p'):
  		heading_tag = soup.find(tag, text=heading)
  		fields = heading_tag.find_next_sibling('p')
  		fields_text = fields.text
  		fields = fields_text.split('\n')
  		for f in fields:
  			#print l
  			if f.find(field) > -1:
  				out_text = f.split(':')[1].strip()
  				return out_text
  		return ''
  
  	def mdata_item(self, mdata_soup, titles, tag_name=None, multiline=False):
  		for title in titles:
  			# Locate the tag containing title
  			tag = bsoup.find_tag_by_text(mdata_soup, title, tag_name)
  
  			#print tag
  			#answer = raw_input("Press enter...")
  
  			if tag is None or len(tag) == 0:
  				continue
  			if multiline:
  				# First, check to see if <pre> is a sibling
  				parent_tag = tag[0].parent
  				pre_tag = parent_tag.find('pre')
  				if pre_tag is None:
  					#print parent_tag
  					sib_tag = parent_tag.next_sibling
  					if sib_tag is None:
  						div_tag = parent_tag.find('div')
  						out_txt = shared.clean_text(div_tag.text)
  					else:
  						out_txt = shared.clean_text(sib_tag.text)
  				else:
  					out_txt = shared.clean_text(pre_tag.text)
  			else:
  				parent_tag = tag[0].parent
  				out_txt = parent_tag.text
  				if out_txt.find(u'\u2003') > -1:
  					text_split = out_txt.split(u'\u2003')
  					out_txt = ' '.join(text_split[1:])
  				else:
  					text_split = out_txt.split(': ')
  					out_txt = ' '.join(text_split[1:])
  
  				#print out_txt
  				#answer = raw_input("Press enter...")
  
  			return out_txt
  
  		return ''
  
  	def get_metadata(self, mdata_url):
  		''' Gets the metadata for a dataset in the MLI pages
  		:param mdata_url: The metadata URL.
  		:return: A dictionary containing the applicable information extract
  					from the metadata page.
  		'''
  
  		mdata_info = collections.OrderedDict()
  
  		# Check the file extension of the metadata URL
  		if mdata_url.find('.zip') > -1 or mdata_url.find('.pdf') > -1:
  			return None
  
  		if mdata_url.find('.txt') > -1:
  			mdata_resp = shared.open_webpage(mdata_url)
  			if isinstance(mdata_resp, int): return 'Metadata link is broken.'
  			mdata_text = mdata_resp.read()
  
  			# Get the description between 'Abstract:' and 'Purpose:'
  			desc_str = ''
  			start_pos = mdata_text.find('Abstract:') + len('Abstract:')
  			end_pos = mdata_text.find('Purpose:')
  			if not start_pos == len('Abstract:') - 1:
  				desc = mdata_text[start_pos:end_pos]
  				desc_str = shared.clean_text(desc)
  				#print desc_str
  
  			# Get the constraints between 'Use_Constraints:' and 'Point_of_Contact:'
  			lic_str = ''
  			start_pos = mdata_text.find('Use_Constraints:') + len('Use_Constraints:')
  			end_pos = mdata_text.find('Point_of_Contact:')
  			if not start_pos == len('Use_Constraints:') - 1:
  				lic = mdata_text[start_pos:end_pos]
  				lic_str = shared.clean_text(lic)
  				#print lic_str
  
  			# Get single line values
  			date_str = ''
  			sp_str = ''
  			mdata_type = ''
  			pub_str = ''
  			m_lines = mdata_text.split('\n')
  			for line in m_lines:
  				if line.find('Calendar_Date') > -1:
  					date_str = line.split(':')[1]
  				elif line.find("Other_Projection's_Definition") > -1:
  					sp_str = line.split(':')[1]
  				elif line.find("Metadata_Standard_Name") > -1:
  					mdata_type = line.split(':')[1]
  				elif line.find("Originator") > -1:
  					pub_str = line.split(':')[1]
  
  			mdata_info['Description'] = desc_str
  			mdata_info['Licensing'] = shared.reduce_text(lic_str)
  			mdata_info['Date'] = date_str
  			mdata_info['Spatial Reference'] = sp_str
  			mdata_info['Metadata Type'] = mdata_type
  			mdata_info['Publisher'] = pub_str
  
  		elif mdata_url.find('.xml') > -1:
  			mdata_xml = bsoup.get_xml_soup(mdata_url)
  			if isinstance(mdata_xml, int): return 'Metadata link is broken.'
  
  			# Get the description
  			desc_str = ''
  			tags = ['abstract', 'idAbs']
  			desc_tag = bsoup.find_xml_tags(mdata_xml, tags)
  			if len(desc_tag) > 0: desc_str = desc_tag[0].text
  
  			# Get the date
  			date_str = ''
  			tags = ['ModDate', 'enddate', 'createDate']
  			date_tag = bsoup.find_xml_tags(mdata_xml, tags)
  			if len(date_tag) > 0: date_str = date_tag[0].text
  
  			# Get the metadata type
  			mdata_type = ''
  			tags = ['metstdn']
  			meta_tag = bsoup.find_xml_tags(mdata_xml, tags)
  			if len(meta_tag) > 0: mdata_type = meta_tag[0].text
  
  			# Get the spatial reference
  			sp_str = ''
  			tags = ['identCode', 'mapprojn']
  			sp_tag = bsoup.find_xml_tags(mdata_xml, tags)
  
  			if len(sp_tag) > 0:
  				sp_str = sp_tag[0].text
  			else:
  				sp_str = 'UTM Zone 14, NAD 83 (EPSG:26914)'
  
  			# Get the publisher
  			pub_str = ''
  			tags = ['origin', 'rpOrgName']
  			pub_tag = bsoup.find_xml_tags(mdata_xml, tags)
  			if len(pub_tag) > 0: pub_str = pub_tag[0].text
  
  			mdata_info['Description'] = desc_str
  			mdata_info['Date'] = date_str
  			mdata_info['Spatial Reference'] = sp_str
  			mdata_info['Metadata Type'] = mdata_type
  			mdata_info['Publisher'] = pub_str
  
  		else:
  			if mdata_url.find('.doc') > -1: return None
  		
  			mdata_soup = bsoup.get_soup(mdata_url)
  			if isinstance(mdata_soup, int): return 'Metadata link is broken.'
  			
  			if mdata_soup is None: return 'Metadata link is broken.'
  			
  			if not self.check_result(mdata_soup, mdata_url, 'MLI Metadata Page'): return None
  
  			html_tag = mdata_soup.find('html')
  			if html_tag is None: return None
  			if isinstance(html_tag, int): return None
  			if html_tag.has_attr('xmlns:res'):
  				# Type 1 Metadata - ArcGIS Metadata
  				# Ex: http://mli2.gov.mb.ca/dems/lidar/Arrow_Oak/Arrow_Oak_LiDAR_DEM_Metadata.htm#ID0EBLA
  
  				# Get the description
  				desc_str = self.mdata_item(mdata_soup, ['Description'], multiline=True)
  
  				# Get the publisher
  
  				# Get the date
  				date_opts = ['Date and time', 'Revision date', 'Publication date']
  				date_str = self.mdata_item(mdata_soup, date_opts)
  
  				# Get the licence
  				lic_opts = ['Use limitations', 'Access and use limitations']
  				lic_str = self.mdata_item(mdata_soup, lic_opts)
  
  				# Get the metadata type
  				type_opts = ['Metadata style', 'Name of the metadata standard used']
  				type_str = self.mdata_item(mdata_soup, type_opts)
  
  				# Get the spatial reference
  				sp_opts = ['Projection']
  				sp_str = self.mdata_item(mdata_soup, sp_opts)
  
  				mdata_info['Description'] = desc_str
  				mdata_info['Licensing'] = lic_str
  				mdata_info['Date'] = date_str
  				mdata_info['Spatial Reference'] = sp_str
  				mdata_info['Metadata Type'] = type_str
  
  			else:
  				# Get the metadata from the meta tags
  				meta_tag_check = mdata_soup.find('meta')
  				if meta_tag_check is None:
  					################################################################
  					# Type 3 Metadata - FGDC & ESRI Metadata
  					# Ex: http://mli2.gov.mb.ca/ortho/meta_files/digital_orthophoto_images_capital_region_meta.html
  
  					# Get the description
  					desc_dt = mdata_soup.find('dt', text='Abstract:')
  					if desc_dt is None: return None
  					desc_pre = desc_dt.find_next_sibling('pre')
  					desc_str = desc_pre.text
  
  					# Get the publisher
  					pub_dt = bsoup.find_tags_containing(mdata_soup, 'Originators:', 'dt')
  					#print "pub_dt: %s" % pub_dt
  					#if isinstance(pub_dt, list): pub_dt = pub_dt[0]
  					pub_str = pub_dt.text.split(":")[1]
  					pub_str = shared.clean_text(pub_str)
  
  					# Get the date
  					#date_dt = bsoup.find_tags_containing(mdata_soup, 'Ending date:', 'dt')
  					date_opts = ['Ending date:', 'Calendar date:']
  					date_str = self.mdata_item(mdata_soup, date_opts)
  
  					# Get the metadata type from the page
  					mtype_dt = bsoup.find_tags_containing(mdata_soup, 'Metadata standard name:', 'dt')
  					mtype_str = mtype_dt.text.split(":")[1]
  					mtype_str = shared.clean_text(mtype_str)
  
  					type_str = ''
  					lic_str = ''
  
  				else:
  					################################################################
  					# Type 2 Metadata - FGDC Metadata
  					# Ex: http://mli2.gov.mb.ca/dems/lidar/south_section/l384_fp_meta.html
  
  					# Get the metadata from the <meta> tags on the page
  					page_mdata = bsoup.get_page_metadata(mdata_soup)
  
  					# Get the description
  					desc_str = ''
  					if 'dc.description' in page_mdata.keys(): desc_str = page_mdata['dc.description']
  
  					# Get the publisher
  					pub_str = ''
  					if 'dc.publisher' in page_mdata.keys(): pub_str = page_mdata['dc.publisher']
  
  					# Get the date
  					date_str = ''
  					if 'dc.date' in page_mdata.keys(): date_str = page_mdata['dc.date']
  
  					# Get the licence
  					lic_str = ''
  					if 'dc.rights' in page_mdata.keys(): lic_str = page_mdata['dc.rights']
  
  					# Get the data type
  					type_str = ''
  					if 'dc.type' in page_mdata.keys(): type_str = page_mdata['dc.type']
  
  					# Get the metadata type from the page
  					mtype_str = ''
  					mtype_tag = bsoup.find_tags_containing(mdata_soup, 'Metadata_Standard_Version', 'em')
  					if mtype_tag is not None:
  						mtype_parent = mtype_tag.parent
  						mtype_txt = shared.clean_text(mtype_parent.text)#.encode('ascii', 'replace'))
  						mtype_str = mtype_txt.split(':')[1]
  
  				mdata_info['Description'] = desc_str
  				mdata_info['Licensing'] = lic_str
  				mdata_info['Date'] = date_str
  				mdata_info['Publisher'] = pub_str
  				mdata_info['Type'] = type_str
  				mdata_info['Metadata Type'] = mtype_str
  
  		#for k, v in mdata_info.items():
  		#	print "%s: %s" % (k, v)
  
  		#answer = raw_input("Press enter...")
  
  		return mdata_info
  
  	def get_world_link(self, soup, url, txt, notes_str=''):
  		''' Gets the world file URL from a soup based on the input txt.
  		:param soup: The soup to perform the search.
  		:param url: The root URL.
  		:param txt: The text contained in the world file's <a> tag.
  		:return: The world file URL.
  		'''
  		world_url = ''
  		notes_str = notes_str
  		world_a = bsoup.find_tags_containing(soup, txt, 'a')
  		if world_a is not None:
  			world_url = shared.get_anchor_url(world_a, url)
  			if notes_str == '':
  				notes_str = "The 'Data URL' contains a link to a Zip file which " \
  							"contains the World file for this dataset."
  
  		return (world_url, notes_str)
  		
  	def process_googleapi(self, layers, lyr_type):
  
  		records = []
  
  		for idx, layer in enumerate(layers):
  			msg = "Extracting %s of %s layers from Google Maps API" % (idx + 1, len(layers))
  			shared.print_oneliner(msg)
  		
  			rec_dict = collections.OrderedDict()
  
  			# if lyr_type == "property":
  			#     element = layer.find('input')
  			#     id = element.get('id')
  			#     id = id.replace("chk", "")
  			# else:
  			#     element = layer
  			#     id = element.get('id')
  			#     id = id.replace("chk-id", "")
  
  			id = layer.get('id')
  			id = id.replace("tbl", "")
  
  			title_str = bsoup.get_text(layer)
  
  			# Build query URL
  			# Ex: https://mapapi.winnipeg.ca/mapapi/wfs.ashx?output=json&maptypeid=2&coordinates=&g=n&featurelist=13937
  			mapapi_url = "https://mapapi.winnipeg.ca/mapapi/wfs.ashx"
  			srch_params = collections.OrderedDict()
  			srch_params['output'] = "json"
  			srch_params['maptypeid'] = "2"
  			srch_params['coordinates'] = ""
  			srch_params['g'] = "n"
  			srch_params['featurelist'] = id
  
  			query_url = shared.get_post_query(mapapi_url, srch_params)
  
  			# Get soup
  			json_res = shared.get_json(query_url)
  
  			features = json_res['features']
  
  			serv_type = features[0]['serviceType']
  			if serv_type == "both": serv_type = "WMS|WFS"
  
  			rec_dict['Title'] = title_str
  			rec_dict['Type'] = serv_type.upper()
  			rec_dict['Description'] = features[0]['DESCRIPTION']
  			rec_dict['Data URL'] = query_url
  
  			records.append(rec_dict)
  
  		return records
  
  	def process_map(self, in_map, page_url, title_prefix=None):
  		''' Goes through a map on a page and creates a dictionary
  				for each <area> tag.
  		:param in_map: The map to process.
  		:param page_url: The URL of the page which contains the map.
  		:param title_prefix: A text to be added to the front of the title.
  		:return: Returns a list of dictionaries for each area.
  		'''
  
  		out_areas = []
  
  		# Get the areas in the map
  		areas = in_map.find_all('area')
  
  		# Cycle through the areas
  		for area in areas:
  
  			# If the area has no 'href' or it is empty, skip it
  			if not area.has_attr('href'): continue
  			if area['href'] == '': continue
  
  			area_dict = collections.OrderedDict()
  
  			# Get the area link
  			area_link = area['href']
  			area_url = urlparse.urljoin(page_url, area_link)
  
  			# Get the title using the 'alt' attribute and the title_prefix
  			if area.has_attr('alt'):
  				area_id = area['alt']
  			else:
  				area_id = os.path.basename(area_url).split('.')[0]
  			if title_prefix is None:
  				title_str = area_id
  			else:
  				title_str = "%s - %s" % (title_prefix, area_id)
  
  			# Add the info to the area_dict
  			area_dict['Title'] = title_str
  			area_dict['Web Page URL'] = page_url
  			area_dict['Download'] = area_url
  			area_dict['Access'] = 'Download/Web Accessible'
  
  			# Get the metadata link
  			if page_url.find('shp') > -1:
  				mdata_map_url = page_url.replace('shp', 'meta')
  				mdata_map_soup = bsoup.get_soup(mdata_map_url)
  				
  				if mdata_map_soup is None:
  					print "\nWARNING: Metadata page '%s' could not be opened." % mdata_map_url
  					print "Please check the 'err_log.csv' file in the province/territory results folder."
  					self.write_error(mdata_map_url, 'Metadata Page', 'Metadata page results could not be opened.')
  					continue
  
  				# Get the actual metadata URL by finding its area's id
  				mdata_area = mdata_map_soup.find('area', attrs={'alt': area_id})
  				mdata_link = mdata_area['href']
  				mdata_url = urlparse.urljoin(mdata_map_url, mdata_link)
  
  				mdata_info = self.get_metadata(mdata_url)
  				if mdata_info is not None:
  					area_dict.update(mdata_info)
  
  				# Get a list of formats
  				formats = []
  				b_doc = mdata_map_soup.find('b', text='.DOC')
  				p_doc = b_doc.parent
  				td_doc = p_doc.parent
  				tr = td_doc.parent
  				td_list = tr.find_all('td')
  				for td in td_list:
  					if not td.text == '.DOC':
  						formats.append(td.text.strip().replace('.', ''))
  
  				# Set the formats, download_str and access_str
  				formats_str = "|".join(formats)
  				download_info = shared.get_download_text(formats)
  				download_str, access_str = download_info.split('|')
  
  				# Add the metadata info to the area_dict
  				area_dict['Available Formats'] = formats_str
  				area_dict['Download'] = download_str
  				area_dict['Access'] = access_str
  				area_dict['Metadata URL'] = mdata_url
  
  			out_areas.append(area_dict)
  
  		return out_areas
  
  	def process_table(self, parent_title, table, index_url, unique_cols=None,
  					  ignore_rows=None, replace_vals=None, title_prefix=None,
  					  start_row=1):
  		''' Processes a table with the first column as the item title and the values
  			in the other columns.
  		:param parent_title: The title of the current parent page.
  		:param table: The table soup object.
  		:param index_url: The URL of the page the table is on.
  		:param unique_cols: The values are specified columns used for specific values.
  							Ex: unique_cols['mdata_col'] = '.doc' means the column with
  								heading '.doc' will only be used for metadata.
  		:param ignore_rows: The values in this list will be used to ignore any rows
  							where the first column is the value.
  		:param replace_vals: A list of tuples where the first value of the tuple
  							will be replaced by the second value in the tuple.
  		:param title_append: A value to append as the start of the title.
  		:return: A list of dictionaries for each row in the table.
  		'''
  		rec_list = []
  
  		# Set up the unique_cols with default values if it is None
  		if unique_cols is None:
  			unique_cols = collections.OrderedDict()
  			unique_cols['mdata_col'] = '.doc'
  			unique_cols['date_col'] = 'date'
  			unique_cols['ignore_cols'] = ['version']
  
  		if not 'ignore_cols' in unique_cols:
  			unique_cols['ignore_cols'] = []
  
  		# Set up ignore_rows with default values if it is None
  		if ignore_rows is None: ignore_rows = ['Go to top']
  
  		# Get the table list
  		table_rows = shared.table_to_dict(table, 0, start_row=start_row)
  		
  		sub_title = ''
  
  		# Go through each row in the table_rows
  		for row in table_rows:
  			# Create the dictionary which will hold the values for the CSV file
  			rec_dict = collections.OrderedDict()
  
  			# Get the title of the dataset from the first column
  			title_tag = row[row.keys()[0]]
  			title = shared.clean_text(title_tag.text.strip())
  
  			# If title is not blank and the first column does not contain any
  			#   of the words in the ignore_rows list
  			#print str(any(s in title for s in ignore_rows))
  			
  			bold = title_tag.find('b')
  			if bold is not None:
  				sub_title = "%s: " % bsoup.get_text(bold)
  				
  			if sub_title == 'Communities': sub_title = ''
  			
  			if not title == "" and not any(s in title for s in ignore_rows):
  				# Remove all whitespace from the title
  				title_str = "%s - %s%s" % (parent_title, sub_title, shared.clean_text(title))
  				formats = []
  				# Get a dictionary of downloads
  				downloads = collections.OrderedDict()
  				# Get the list of formats from all remaining columns
  				#   which are not in the unique_cols dictionary
  
  				dtype = ''
  				for key, value in row.items()[1:]:
  					key_text = shared.clean_text(key)
  					if isinstance(value, str):
  						val_text = shared.clean_text(value)
  					else:
  						val_text = shared.clean_text(value.text)
  					#print "title_str: '%s'" % title_str
  					#print "key_text: " + str(key_text)
  					#print "val_text: " + str(val_text)
  					if replace_vals is not None:
  						for v in replace_vals:
  							if val_text == v[0]:
  								val_text = v[1]
  
  					if not key_text in unique_cols.values() and \
  							not key_text in unique_cols['ignore_cols'] and \
  							not val_text == 'Link' and \
  							not val_text == "":
  						# Get the formats
  						format = shared.clean_text(val_text)
  						if format.find(u'\xa0') > -1:
  							format_split = format.split(u'\xa0 ')
  							for form in format_split:
  								formats.append(form.replace('.', ''))
  						else:
  							formats.append(format)
  						# Get the download list
  						if not isinstance(value, str):
  							a = value.find('a')
  							if a is not None:
  								url = shared.get_anchor_url(a, index_url)
  								frm = shared.clean_text(a.text.lower())
  								if replace_vals is not None:
  									for v in replace_vals:
  										if frm == v[0].lower():
  											frm = v[1]
  								frm = frm.replace('.', '')
  								downloads[frm] = url
  								dtype = shared.get_data_type(url)
  
  				#print formats
  
  				# Get the proper download and access text
  				if len(downloads.keys()) == 0:
  					download_str = 'No'
  					access = 'Contact the Province'
  				else:
  					if len(formats) == 0:
  						download_str = 'No'
  						access = 'Contact the Province'
  					elif len(formats) == 1:
  						#print downloads
  						#print formats
  						download_str = downloads[downloads.keys()[0]]
  						access = 'Download/Web Accessible'
  					else:
  						download_str = 'Multiple Downloads'
  						access = 'Download/Web Accessible'
  
  				# Get the metadata URL
  				doc_soup = None
  				mdata_url = ''
  				if 'mdata_col' in unique_cols:
  					mdata_col = unique_cols['mdata_col']
  					if mdata_col in row:
  						doc_soup = row[mdata_col]
  					elif 'metadata' in row:
  						doc_soup = row['metadata']
  
  					if doc_soup is not None and not isinstance(doc_soup, str):
  						mdata_a = doc_soup.find('a')
  						#print mdata_a
  						if mdata_a is not None:
  							mdata_url = shared.get_anchor_url(mdata_a, index_url)
  
  						# Get the metadata info
  						if mdata_url is not None and not mdata_url == '':
  							mdata_info = self.get_metadata(mdata_url)
  							if mdata_info is not None:
  								rec_dict.update(mdata_info)
  
  				# Get the description, if applicable
  				desc_str = ''
  				if 'desc_col' in unique_cols:
  					desc_col = unique_cols['desc_col']
  					if desc_col in row:
  						doc_soup = row[desc_col]
  
  					if doc_soup is not None and not isinstance(doc_soup, str):
  						desc_str = doc_soup.text
  
  				# Get the date, if applicable
  				date_str = ''
  				if 'date_col' in unique_cols:
  					date_col = unique_cols['date_col']
  					if date_col in row:
  						date_soup = row[date_col]
  						if isinstance(date_soup, str):
  							date_str = date_soup.strip()
  						else:
  							date_str = date_soup.text.strip()
  
  				# Get the spatial reference, if applicable
  				sp_str = 'UTM Zone 14, NAD 83 (EPSG:26914)'
  				if 'ref_col' in unique_cols:
  					ref_col = unique_cols['ref_col']
  					if ref_col in row:
  						sp_soup = row[ref_col]
  						if isinstance(sp_soup, str):
  							sp_str = sp_soup.strip()
  						else:
  							sp_str = sp_soup.text.strip()
  
  				# Append the title if applicable
  				if title_prefix is not None:
  					title_str = "%s - %s" % (title_prefix, title_str)
  
  				# Store the vales in the dictionary
  				rec_dict['Title'] = title_str
  				rec_dict['Available Formats'] = "|".join([f.replace('.', '') for f in formats])
  				rec_dict['Access'] = access
  				rec_dict['Type'] = dtype
  				rec_dict['Download'] = download_str
  				rec_dict['Metadata URL'] = mdata_url
  				rec_dict['Spatial Reference'] = sp_str
  				rec_dict['Web Page URL'] = index_url
  				rec_dict['Date'] = date_str
  
  				# FOR DEBUG ONLY
  				#for k, v in rec_dict.items():
  				#	print "%s: %s" % (k, v)
  
  				# Add the record dictionary to the list of records
  				rec_list.append(rec_dict)
  
  		return rec_list
  
  	def extract_dem(self):
  		''' Extracts the DEM downloads
  		:return: None
  		'''
  		
  		self.print_title("Extracting Manitoba's DEM page")
  		
  		start_time = datetime.datetime.now()
  		print "Process started at: %s" % start_time
  
  		# Get the DEM main url
  		main_url = self.pg_grp.get_url('main_url')
  		mdata_url = self.pg_grp.get_url('mdata_url')
  
  		# Create the CSV file
  		csv_fn = "DEM_results"
  		pt_csv = sh.PT_CSV(csv_fn, self)
  		pt_csv.open_csv()
  
  		# First get the metadata info
  		mdata_soup = bsoup.get_soup(mdata_url)
  		
  		if mdata_soup is None:
  			print "\nWARNING: DEM metadata page '%s' could not be opened." % mdata_url
  			print "Please check the 'err_log.csv' file in the province/territory results folder."
  			self.write_error(mdata_url, 'DEM Metadata Page', 'DEM metadata page results could not be opened.')
  			return None
  
  		# Split the page at <h2> tags in order to separate the different datasets
  		h2_list = str(mdata_soup).split('<h2')
  
  		# Loop through each dataset on the page and add the information to a
  		#   dictionary containing the <h2> text as keys
  		mdata_dict = collections.OrderedDict()
  		for h2 in h2_list[1:]:
  
  			sub_soup = BeautifulSoup("<h2" + h2, 'html.parser')
  
  			# Get the title to store in the mdata_dict
  			h2_text = sub_soup.find('h2').text
  			title = h2_text.split(u'\u2013')[0].strip()
  			if title.find("Manitoba") > -1:
  				title = title.replace(' Version', '')
  
  			#print title
  
  			# Get the description text
  			desc_h = bsoup.find_tags_containing(sub_soup, 'Description', 'h4')
  			if desc_h is None:
  				desc_h = bsoup.find_tags_containing(sub_soup, 'Description', 'h5')
  			desc_p = desc_h.find_next_sibling('p')
  			desc_str = bsoup.get_text(desc_p).replace('Abstract: ', '')
  
  			sp_str = 'UTM Zone 14, NAD 1983'
  
  			# Get the data from the publication date
  			date = self.get_mdata_item(sub_soup, 'Citation Information: ', 'Publication Date: ')
  			date_str = datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')
  
  			pub_str = self.get_mdata_item(sub_soup, 'Citation Information: ', 'Publisher: ')
  
  			#print desc_str
  			#print sp_str
  			#print date_str
  			#print pub_str
  
  			info = collections.OrderedDict()
  			info['Description'] = desc_str
  			info['Spatial Reference'] = sp_str
  			info['Date'] = date_str
  			info['Publisher'] = pub_str
  
  			mdata_dict[title] = info
  
  		# Get the soup
  		dem_soup = bsoup.get_soup(main_url)
  		
  		if dem_soup is None:
  			print "\nWARNING: DEM main page '%s' could not be opened." % main_url
  			print "Please check the 'err_log.csv' file in the province/territory results folder."
  			self.write_error(main_url, 'DEM Main Page', 'Page could not be opened.')
  			return None
  
  		# Find the h2 containing the word 'Ordering'
  		h2 = bsoup.find_tags_containing(dem_soup, 'Ordering', 'h2')
  		ol = h2.find_next_sibling('ol')
  		heading_list = ol.find_all('li')
  
  		# Build a collection of unique datasets from the heading list
  		# Ex:
  		#   ds_dict =
  		#   "DEMSM Version 1":
  		#   {
  		#	    "formats": [
  		#           "ASCII Grid",
  		#           "MapInfo Grid",
  		#           "Tiff Image"
  		#   	],
  		#	    "links": [
  		#           "#1",
  		#           "#2",
  		#           "#3"
  		#	    ]
  		#   },
  		#   ...
  		ds_dict = collections.OrderedDict()
  		for h in heading_list:
  			# Parse the heading text
  			a = h.find('a')
  			h_text = a.text
  			end_pos = h_text.find('(')
  			title_str = h_text[:end_pos].strip()
  			if title_str in ds_dict.keys():
  				info_dict = ds_dict[title_str]
  			else:
  				info_dict = collections.OrderedDict()
  				info_dict['formats'] = []
  				info_dict['links'] = []
  			formats = info_dict['formats']
  			format = shared.get_bracket_text(h_text)
  			formats.append(format)
  			info_dict['formats'] = formats
  			links = info_dict['links']
  			links.append(a['href'])
  			info_dict['links'] = links
  
  			ds_dict[title_str] = info_dict
  
  		for k, v in ds_dict.items():
  			idx = ds_dict.keys().index(k)
  			msg = "Extracting %s of %s datasets" % (idx + 1, len(ds_dict.items()))
  			shared.print_oneliner(msg)
  		
  			formats = v['formats']
  			links = v['links']
  
  			title_str = k
  			access_str = 'Download/Web Accessible'
  			available_formats = '|'.join([f.replace(" Grid", "") for f in formats])
  			if len(links) > 1:
  				download_str = 'Multiple Downloads'
  			else:
  				download_str = ''
  				bookmark = links[0].replace("#", "")
  				# Locate <a> with bookmark ID
  				a_bkmark = dem_soup.find('a', attrs={'id': bookmark})
  				li_parent = a_bkmark.parent
  				a_list = li_parent.find_all('a')
  				for a in a_list:
  					if a.has_attr('href'):
  						download_str = urlparse.urljoin(main_url, a['href'])
  
  			mdata_key = title_str.split('-')[0]
  			mdata_key = mdata_key.strip()
  			mdata_key = mdata_key.replace('DEMSM ', '')
  			if mdata_key in mdata_dict:
  				mdata_info = mdata_dict[mdata_key]
  
  				for mdata_k, mdata_v in mdata_info.items():
  					pt_csv.add(mdata_k, mdata_v)
  
  			# Add all geospatial data to the pt_csv
  			pt_csv.add('Source', 'Manitoba DEMs')
  			pt_csv.add('Title', title_str)
  			pt_csv.add('Access', access_str)
  			pt_csv.add('Type', 'Raster Data')
  			pt_csv.add('Download', download_str)
  			pt_csv.add('Web Page URL', main_url)
  			pt_csv.add('Metadata URL', mdata_url)
  			pt_csv.add('Available Formats', available_formats)
  
  			pt_csv.write_dataset()
  
  		pt_csv.close_csv()
  		
  		# Print ending time
  		end_time = datetime.datetime.now()
  		print "\nExtraction complete at %s." % end_time
  		tot_time = end_time - start_time
  		print "It took %s to complete." % tot_time
  
  	def extract_maps(self):
  		''' Method to extract all interactive maps for Manitoba.
  		:return: None
  		'''
  
  		self.print_log("\nExtracting from %s" % self.pg_grp.get_title())
  		
  		self.print_title("Extracting Manitoba's interactive maps")
  		
  		start_time = datetime.datetime.now()
  		print "Process started at: %s" % start_time
  
  		# Create the CSV file
  		csv_fn = "InteractiveMaps_results"
  		pt_csv = sh.PT_CSV(csv_fn, self)
  		pt_csv.open_csv()
  		
  		cur_page = 0
  		page_count = self.pg_grp.get_page_count()
  
  		################################################################################
  		# Start with the Petroleum map
  		petrol_parent_url = self.pg_grp.get_url('petrol_url')[0]
  		petrol_map_url = self.pg_grp.get_url('petrol_url')[1]
  		petrol_service_url = self.pg_grp.get_url('petrol_url')[2]
  
  		# Get the info from the parent page
  		petrol_parent_soup = bsoup.get_soup(petrol_parent_url)
  		
  		if self.check_result(petrol_parent_soup, petrol_parent_url, 'Petroleum Map'):
  			# Print the status
  			cur_page += 1
  			msg = "Extracting web map %s of %s maps" % (cur_page, page_count)
  			shared.print_oneliner(msg)
  		
  			# Get the main-content <div>
  			content_div = petrol_parent_soup.find('div', attrs={'id': 'main-content'})
  
  			# Get the title from the main-content <div>
  			title_h = content_div.find('h1')
  			title_str = bsoup.get_text(title_h)
  
  			# Get the description from the parent page
  			desc_tag = bsoup.find_tags_containing(petrol_parent_soup, 'this Interactive Map', 'strong')
  			desc_str = bsoup.get_text(desc_tag)
  
  			# Get the date from the parent page
  			date_tag = bsoup.find_tags_containing(petrol_parent_soup, 'Updated:', 'strong')
  			date_str = bsoup.get_text(date_tag)
  
  			sp_str = 'UTM Zone 14, NAD 83 (EPSG:26914)'
  
  			# Add all values to the CSV file object
  			pt_csv.add('Source', 'Manitoba Interactive Maps')
  			pt_csv.add('Title', title_str)
  			pt_csv.add('Description', desc_str)
  			pt_csv.add('Web Map URL', petrol_map_url)
  			pt_csv.add('Data URL', petrol_service_url)
  			pt_csv.add('Type', 'Web Map Application')
  			pt_csv.add('Date', date_str)
  			pt_csv.add('Access', 'Viewable/Contact the Province')
  			pt_csv.add('Download', 'No')
  			pt_csv.add('Spatial Reference', sp_str)
  
  			# Write the dataset to the CSV file
  			pt_csv.write_dataset()
  
  		################################################################################
  		# Drought Management Map
  		drought_url = self.pg_grp.get_url('drought_url')
  
  		# Get the ArcGIS Online data URL from the <iframe> tag on the drought page
  		drought_soup = bsoup.get_soup(drought_url)
  		
  		if self.check_result(drought_soup, drought_url, 'Drought Management Map'):
  			# Print the status
  			cur_page += 1
  			msg = "Extracting web map %s of %s maps" % (cur_page, page_count)
  			shared.print_oneliner(msg)
  		
  			drought_iframe = drought_soup.find('iframe')
  			drought_datasrc = drought_iframe['src']
  
  			# Get the ArcGIS Online data
  			data_url = shared.get_arcgis_url(drought_datasrc)
  
  			map_json = shared.get_json(data_url)
  			
  			# If map_json is None
  			if self.check_result(map_json, data_url, 'Drought Management Map JSON Results'):
  				# Get the title, date and data type
  				title_str = map_json['title']
  				date_str = shared.translate_date(map_json['modified'])
  				dtype = map_json['type']
  
  				# Get the description
  				desc_html = map_json['description']
  				desc_soup = BeautifulSoup(desc_html, 'html.parser')
  				desc_str = bsoup.get_text(desc_soup)
  
  				# Get the map URL
  				map_url = map_json['url']
  
  				# Get the licence info
  				lic_html = map_json['licenseInfo']
  				lic_soup = BeautifulSoup(lic_html, 'html.parser')
  				lic_str = bsoup.get_text(lic_soup)
  
  				# Add all values to the CSV file object
  				pt_csv.add('Source', 'Manitoba Interactive Maps')
  				pt_csv.add('Title', title_str)
  				pt_csv.add('Description', desc_str)
  				pt_csv.add('Web Map URL', map_url)
  				pt_csv.add('Web Page URL', drought_url)
  				pt_csv.add('Data URL', data_url)
  				pt_csv.add('Type', dtype)
  				pt_csv.add('Date', date_str)
  				pt_csv.add('Licensing', lic_str)
  				pt_csv.add('Access', 'Viewable/Contact the Province')
  				pt_csv.add('Download', 'No')
  				#pt_csv.add('Spatial Reference', sp_str)
  
  				# Write the dataset to the CSV file
  				pt_csv.write_dataset()
  
  		################################################################################
  		# Drinking Water Advisories in Manitoba Map
  		drink_url = self.pg_grp.get_url('drink_url')
  
  		# Get the ArcGIS Online data URL from the <iframe> tag on the drought page
  		drink_soup = bsoup.get_soup(drink_url)
  		
  		if self.check_result(drink_soup, drink_url, 'Drinking Water Advisories in Manitoba Map'):
  			# Print the status
  			cur_page += 1
  			msg = "Extracting web map %s of %s maps" % (cur_page, page_count)
  			shared.print_oneliner(msg)
  		
  			drink_iframe = drink_soup.find('iframe')
  			drink_datasrc = drink_iframe['src']
  
  			# Get the ArcGIS Online data
  			data_url = shared.get_arcgis_url(drink_datasrc)
  
  			map_json = shared.get_json(data_url)
  			
  			if self.check_result(map_json, data_url, 'Drinking Water Advisories in Manitoba Map JSON Results'):
  				# Get the title, date and data type
  				title_str = map_json['title']
  				date_str = shared.translate_date(map_json['modified'])
  				dtype = map_json['type']
  
  				# Get the description
  				desc_html = map_json['description']
  				desc_soup = BeautifulSoup(desc_html, 'html.parser')
  				desc_str = bsoup.get_text(desc_soup)
  
  				# Get the map URL
  				map_url = map_json['url']
  
  				# Get the licence info
  				lic_html = map_json['licenseInfo']
  				lic_soup = BeautifulSoup(lic_html, 'html.parser')
  				lic_str = bsoup.get_text(lic_soup)
  
  				# Add all values to the CSV file object
  				pt_csv.add('Source', 'Manitoba Interactive Maps')
  				pt_csv.add('Title', title_str)
  				pt_csv.add('Description', desc_str)
  				pt_csv.add('Web Map URL', map_url)
  				pt_csv.add('Web Page URL', drought_url)
  				pt_csv.add('Data URL', data_url)
  				pt_csv.add('Type', dtype)
  				pt_csv.add('Date', date_str)
  				pt_csv.add('Licensing', lic_str)
  				pt_csv.add('Access', 'Viewable/Contact the Province')
  				pt_csv.add('Download', 'No')
  				# pt_csv.add('Spatial Reference', sp_str)
  
  				# Write the dataset to the CSV file
  				pt_csv.write_dataset()
  
  		################################################################################
  		# WeatherView map
  
  		wview_map_url = self.pg_grp.get_url('weather_url')[0]
  		wview_data_url = self.pg_grp.get_url('weather_url')[1]
  
  		# Get the soup of the map page
  		wview_soup = bsoup.get_soup(wview_map_url)
  		
  		if self.check_result(wview_soup, wview_map_url, 'WeatherView Map'):
  			# Print the status
  			cur_page += 1
  			msg = "Extracting web map %s of %s maps" % (cur_page, page_count)
  			shared.print_oneliner(msg)
  		
  			# Get the main-content <div>
  			content_div = wview_soup.find('div', attrs={'id': 'main-content'})
  
  			# Get the title from the main-content <div>
  			title_h = content_div.find('h1')
  			title_str = bsoup.get_text(title_h)
  
  			# Get the description
  			desc_p = title_h.find_next_sibling('p')
  			desc_str = bsoup.get_text(desc_p)
  
  			# Add all values to the CSV file object
  			pt_csv.add('Source', 'Manitoba Interactive Maps')
  			pt_csv.add('Title', title_str)
  			pt_csv.add('Description', desc_str)
  			pt_csv.add('Web Map URL', wview_map_url)
  			pt_csv.add('Web Page URL', wview_map_url)
  			pt_csv.add('Data URL', wview_data_url)
  			pt_csv.add('Type', 'Web Map Application')
  			pt_csv.add('Access', 'Viewable/Contact the Province')
  			pt_csv.add('Download', 'No')
  
  			# Write the dataset to the CSV file
  			pt_csv.write_dataset()
  
  		################################################################################
  		# Fire View map
  
  		fireview_url = self.pg_grp.get_url('fireview_url')[0]
  		parent_url = self.pg_grp.get_url('fireview_url')[1]
  
  		# Get the soup for the FireView map
  		fireview_soup = bsoup.get_soup(fireview_url)
  		
  		if self.check_result(fireview_soup, fireview_url, 'Fire View Map'):
  			# Print the status
  			cur_page += 1
  			msg = "Extracting web map %s of %s maps" % (cur_page, page_count)
  			shared.print_oneliner(msg)
  		
  			# Get the main-content <div>
  			content_div = fireview_soup.find('div', attrs={'id': 'main-content'})
  
  			# Get the title from the main-content <div>
  			title_h = content_div.find('h1')
  			title_str = bsoup.get_text(title_h)
  
  			# Get the sibling <p> to get the description of the map
  			desc_p = title_h.find_next_sibling('p')
  			desc_str = bsoup.get_text(desc_p)
  
  			# Add all values to the CSV file object
  			pt_csv.add('Source', 'Manitoba Interactive Maps')
  			pt_csv.add('Title', title_str)
  			pt_csv.add('Description', desc_str)
  			pt_csv.add('Web Map URL', fireview_url)
  			pt_csv.add('Web Page URL', parent_url)
  			pt_csv.add('Type', 'Web Map Application')
  			pt_csv.add('Access', 'Viewable/Contact the Province')
  			pt_csv.add('Download', 'No')
  
  			# Write the dataset to the CSV file
  			pt_csv.write_dataset()
  
  			################################################################################
  			# Fire Restrictions map
  
  			restrict_url = self.pg_grp.get_url('restrict_url')
  
  			title_str = 'Fire Restrictions Map'
  
  			# Get the main-content <div>
  			content_div = fireview_soup.find('div', attrs={'id': 'main-content'})
  
  			# Get the sibling <p> to get the description of the map
  			desc_p = content_div.find_next_sibling('p')
  			desc_str = bsoup.get_text(desc_p)
  
  			# Add all values to the CSV file object
  			pt_csv.add('Source', 'Manitoba Interactive Maps')
  			pt_csv.add('Title', title_str)
  			pt_csv.add('Description', desc_str)
  			pt_csv.add('Web Map URL', restrict_url)
  			pt_csv.add('Type', 'Web Map Application')
  			pt_csv.add('Access', 'Viewable/Contact the Province')
  			pt_csv.add('Download', 'No')
  
  			# Write the dataset to the CSV file
  			pt_csv.write_dataset()
  
  		################################################################################
  		# Wildlife Management Areas map
  
  		wildmap_url = self.pg_grp.get_url('wild_url')[1]
  		home_url = self.pg_grp.get_url('wild_url')[0]
  
  		# Get the soup of the home page
  		home_soup = bsoup.get_soup(home_url)
  		
  		if self.check_result(home_soup, home_url, 'Wildlife Management Areas Map'):
  			# Print the status
  			cur_page += 1
  			msg = "Extracting web map %s of %s maps" % (cur_page, page_count)
  			shared.print_oneliner(msg)
  		
  			# Get the body <div>
  			body_div = home_soup.find('div', attrs={'class': 'body'})
  
  			# Get the title from the <h2> tag in the body_div
  			title_h2 = body_div.find('h2')
  			title_str = bsoup.get_text(title_h2)
  
  			# Get the description from the <p> tags from the body_div
  			p_list = body_div.find_all('p')
  			desc_str = ''.join(bsoup.get_text(p) for p in p_list)
  
  			# Add all values to the CSV file object
  			pt_csv.add('Source', 'Manitoba Interactive Maps')
  			pt_csv.add('Title', title_str)
  			pt_csv.add('Description', desc_str)
  			pt_csv.add('Web Map URL', wildmap_url)
  			pt_csv.add('Web Page URL', home_url)
  			pt_csv.add('Type', 'Web Map Application')
  			pt_csv.add('Access', 'Viewable/Contact the Province')
  			pt_csv.add('Download', 'No')
  
  			# Write the dataset to the CSV file
  			pt_csv.write_dataset()
  
  		################################################################################
  		# Beach Monitoring Information map
  
  		beach_url =  self.pg_grp.get_url('beach_url')[0]
  		beachmap_url = self.pg_grp.get_url('beach_url')[1]
  
  		# Get the soup of the main page
  		beach_soup = bsoup.get_soup(beach_url)
  		
  		if self.check_result(beach_soup, beach_url, 'Beach Monitoring Information Map'):
  			# Print the status
  			cur_page += 1
  			msg = "Extracting web map %s of %s maps" % (cur_page, page_count)
  			shared.print_oneliner(msg)
  		
  			# Get the body <div>
  			body_div = beach_soup.find('div', attrs={'class': 'body'})
  
  			# Get the description from the body div by getting the first <p>
  			desc_p = body_div.find('p')
  			desc_str = bsoup.get_text(desc_p)
  
  			# Now get the soup for the beach map
  			beachmap_soup = bsoup.get_soup(beachmap_url)
  
  			# Get the title string from the <h1> in the main <div>
  			main_div = beachmap_soup.find('div', attrs={'id': 'main-content'})
  			title_h1 = main_div.find('h1')
  			title_str = bsoup.get_text(title_h1)
  
  			# Add all values to the CSV file object
  			pt_csv.add('Source', 'Manitoba Interactive Maps')
  			pt_csv.add('Title', title_str)
  			pt_csv.add('Description', desc_str)
  			pt_csv.add('Web Map URL', beachmap_url)
  			pt_csv.add('Web Page URL', beach_url)
  			pt_csv.add('Type', 'Web Map Application')
  			pt_csv.add('Access', 'Viewable/Contact the Province')
  			pt_csv.add('Download', 'No')
  
  			# Write the dataset to the CSV file
  			pt_csv.write_dataset()
  			
  		print
  
  		pt_csv.close_csv()
  		
  		# Print ending time
  		end_time = datetime.datetime.now()
  		print "\nExtraction complete at %s." % end_time
  		tot_time = end_time - start_time
  		print "It took %s to complete." % tot_time
  
  	def extract_misc(self):
  		''' Extracts the miscellaneous web pages
  		:return: None
  		'''
  		
  		self.print_title("Extracting Manitoba's other web pages")
  		
  		start_time = datetime.datetime.now()
  		print "Process started at: %s" % start_time
  
  		##########################################################################
  		# Get the Petroleum page
  
  		# Get the petroleum main url
  		main_url = self.pg_grp.get_url('petrol_url')
  
  		# Create the CSV file
  		csv_fn = "Misc_results"
  		pt_csv = sh.PT_CSV(csv_fn, self)
  		pt_csv.open_csv()
  		
  		cur_page = 0
  		page_count = self.pg_grp.get_page_count()
  
  		# Get the soup for petroleum site
  		petrol_soup = bsoup.get_soup(main_url)
  		
  		if self.check_result(petrol_soup, main_url, 'Petroleum Page'):
  			# Print the status
  			cur_page += 1
  			msg = "Extracting web page %s of %s maps" % (cur_page, page_count)
  			shared.print_oneliner(msg)
  		
  			# Find the <h2> with 'GIS Shapefiles to Download'
  			down_h2 = petrol_soup.find('h2', text='GIS Shapefiles to Download')
  		
  			# Find its sibling <div>
  			down_div = down_h2.find_next_sibling('div')
  			# Now get the download anchors
  			a_list = down_div.find_all('a')
  
  			for a in a_list:
  				download_url = urlparse.urljoin(main_url, a['href'])
  				access_str = 'Download/Web Accessible'
  				title_str = a.text.replace(' (zip)', '')
  				date_str = a.next_sibling
  				sp_str = 'UTM Zone 14, NAD 1983'
  				type_str = 'Vector Data'
  
  				# Add all geospatial data to the pt_csv
  				pt_csv.add('Source', 'Manitoba Web Pages')
  				pt_csv.add('Title', title_str)
  				# pt_csv.add('Description', desc_str)
  				pt_csv.add('Access', access_str)
  				pt_csv.add('Type', type_str)
  				pt_csv.add('Download', download_url)
  				pt_csv.add('Spatial Reference', sp_str)
  				pt_csv.add('Web Page URL', main_url)
  				pt_csv.add('Date', date_str)
  				pt_csv.add('Available Formats', 'SHP')
  
  				pt_csv.write_dataset()
  
  		##########################################################################
  		# Extract the Fire page
  
  		# Get the soup for the fire page
  		fire_url = self.pg_grp.get_url('fire_url')
  		fire_soup = bsoup.get_soup(fire_url)
  		
  		if self.check_result(fire_soup, fire_url, 'Fire Page'):
  			# Print the status
  			cur_page += 1
  			msg = "Extracting web page %s of %s maps" % (cur_page, page_count)
  			shared.print_oneliner(msg)
  		
  			# Find the <div> with id 'main-content'
  			main_div = fire_soup.find('div', attrs={'id': 'main-content'})
  
  			# Get a list of all the links in the <div>
  			a_list = main_div.find_all('a')
  
  			# Add links to a dictionary with the link's year as a key
  			ds = collections.OrderedDict()
  			for a in a_list:
  				a_text = bsoup.get_text(a)
  
  				year = re.findall('(\d{4})', a_text)
  
  				if len(year) > 0:
  					if year[0] not in ds.keys():
  						ds[year[0]] = []
  
  					ds[year[0]].append(a)
  
  			# Loop through each year in the dictionary
  			#   and add each link to the CSV file
  			for k, v in ds.items():
  
  				# Get the title
  				title_str = 'Fire Locations - %s' % k
  
  				# Determine the formats
  				formats = []
  				for a in v:
  					a_link = a['href']
  					if a_link.lower().find('.zip') > -1:
  						#print a_link
  						formats.append('SHP')
  					elif a_link.lower().find('.kmz') > -1:
  						#print a_link
  						formats.append('KMZ')
  
  				# Get the URL of the link
  				download_url = shared.get_anchor_url(v[0], fire_url)
  
  				# Determine the download_str and access_str
  				download_info = shared.get_download_text(formats, download_url)
  				download_str, access_str = download_info.split('|')
  
  				# Add all geospatial data to the pt_csv
  				pt_csv.add('Source', 'Manitoba Web Pages')
  				pt_csv.add('Title', title_str)
  				pt_csv.add('Access', access_str)
  				pt_csv.add('Type', 'Vector Data')
  				pt_csv.add('Download', download_str)
  				pt_csv.add('Web Page URL', main_url)
  				pt_csv.add('Date', k)
  				pt_csv.add('Available Formats', "|".join(formats))
  
  				pt_csv.write_dataset()
  
  			fireidx_url = self.pg_grp.get_url('fireidx_url')
  			fireidx_soup = bsoup.get_soup(fireidx_url)
  			
  			if self.check_result(fireidx_soup, fireidx_url, 'Fire Index Page'):
  				file_table = fireidx_soup.find('table')
  				file_rows = shared.table_to_dict(file_table)
  
  				for row in file_rows:
  					for k, v in row.items():
  						if isinstance(v, str):
  							v_text = v
  						else:
  							v_text = v.text
  						#print "%s: %s" % (k, v_text)
  
  		pt_csv.close_csv()
  		
  		# Print ending time
  		end_time = datetime.datetime.now()
  		print "\nExtraction complete at %s." % end_time
  		tot_time = end_time - start_time
  		print "It took %s to complete." % tot_time
  
  	def extract_mli(self): #, subpage='all'):
  		''' Extract all the different sites of Manitoba Land Initiative
  			:return: None
  		'''
  		
  		# Get the parameters
  		subpage = self.get_arg_val('subpage')
  		
  		self.print_title("Extracting Manitoba's Land Initiative web pages")
  		
  		start_time = datetime.datetime.now()
  		print "Process started at: %s" % start_time
  
  		# NOTE: subpage is for debugging only.
  		if subpage is None: subpage = 'all'
  		# subpage = 'all'
  
  		index_url = self.pg_grp.get_url('main_url')
  
  		# Get the section DIV
  		#divs = soup.find_all('div', attrs={'class': 'section'})
  
  		# Remove <br> between Land Use Maps and Municipal Maps
  		idx_page = shared.open_webpage(index_url)
  		before_txt = 'Land Use/Cover Maps</a><br>'
  		after_txt = "Land Use/Cover Maps</a></li><li>"
  		idx_content = idx_page.read()
  		#print idx_content.find(before_txt)
  		idx_content = idx_content.replace(before_txt, after_txt)
  
  		soup = BeautifulSoup(idx_content, 'html.parser')
  		
  		if not self.check_result(soup, index_url, 'Manitoba Land Initiative Page'):
  			return None
  
  		divs = soup.find_all('div', attrs={'class': 'section'})
  
  		#print divs
  
  		# Get the li
  		li_list = divs[0].find_all('li')
  
  		#print "Number of records: %s" % str(len(li_list))
  
  		mli_folder = "%s\\MLI_Results" % self.work_folder
  		if not os.path.exists(mli_folder):
  			os.mkdir(mli_folder)
  
  		for idx, li in enumerate(li_list):
  
  			#a_list = li.find_all('a')
  			anchor = li.find('a')
  
  			#for anchor in a_list:
  			item_str = bsoup.get_text(anchor)
  
  			# Ignore about and comments
  			if item_str == 'About': continue
  			if item_str == 'Comments': continue
  
  			# Get the item's link and title
  			page_url = anchor['href']
  			parent_title = shared.clean_text(anchor.string)
  
  			# Check to see if the subpage matches the title
  			page_id = self.mli_opts[parent_title]
  			#print parent_title
  			#print page_id
  			if not subpage == 'all' and not subpage == page_id:
  				continue
  
  			# Build the page's URL
  			if page_url.find("../../") > -1:
  				page_url = page_url.replace("../", "", 1)
  			page_url = urlparse.urljoin(index_url, page_url)
  			
  			print "\nExtracting Category: " + str(parent_title)
  			print "Category URL: " + str(page_url)
  
  			# Set up the attributes variable for the selenium
  			if item_str.lower() == "digital imagery":
  				selenium = True
  				attr_name = ['div', 'addthis_toolbox']
  			else:
  				selenium = False
  				attr_name = None
  			
  			# Get the category soup
  			rec_soup = bsoup.get_soup(page_url, selenium, attr_name)
  				
  			if not self.check_result(rec_soup, page_url, 'Manitoba Land Initiative - %s Page' % parent_title): continue
  
  			# Reformat the parent_title for the CSV filename
  			fn_tag = shared.clean_text(parent_title)
  			fn_tag = fn_tag.replace(" ", "-").replace("/", "-").replace("\\", "-")
  
  			sp_str = 'UTM Zone 14, NAD 83 (EPSG:26914)'
  			
  			source = 'Manitoba MLI - %s' % parent_title
  
  			if parent_title == 'Administrative Boundaries' or parent_title == 'Base Maps' or \
  					parent_title == 'Environment' or parent_title == 'Geographical Names' or \
  					parent_title == 'Land Use/Cover Maps' or parent_title == 'Transportation':
  
  				# Create the CSV file and determining category
  				csv_fn = "MLI_%s_results" % fn_tag
  				pt_csv = sh.PT_CSV(csv_fn, self)
  				pt_csv.open_csv()
  
  				# These sites include standard tables with no other links
  				#   on the page
  				table_list = rec_soup.find_all('table')
  				for table in table_list:
  				
  					# Set the unique_cols
  					unique_cols = collections.OrderedDict()
  					unique_cols['mdata_col'] = '.doc'
  					unique_cols['ignore_cols'] = ['.image', '.gif', 'version']
  					replace_vals = [('GEO', 'FGDB')]
  					rec_list = self.process_table(parent_title, table, page_url, replace_vals=replace_vals)
  
  					for idx, rec in enumerate(rec_list):
  						msg = "Extracting %s of %s records of %s" % (idx + 1, len(rec_list), parent_title)
  						shared.print_oneliner(msg)
  						for k, v in rec.items():
  							pt_csv.add(k, v)
  
  						pt_csv.add('Source', source)
  						pt_csv.write_dataset()
  						
  					print
  
  				pt_csv.close_csv()
  
  			elif parent_title == 'Cadastral':
  
  				# Create the CSV file and determining category
  				csv_fn = "MLI_%s_results" % fn_tag
  				pt_csv = sh.PT_CSV(csv_fn, self)
  				pt_csv.open_csv()
  
  				# The Cadastral site includes a single table for the geodatabase
  				#   dataset and a few links to other datasets and indices
  
  				###########################################################
  				# Start with table on the Cadastral page
  				table_list = rec_soup.find_all('table')
  				table = table_list[1]
  				unique_cols = collections.OrderedDict()
  				unique_cols['mdata_col'] = 'metadata'
  				unique_cols['date_col'] = 'date'
  				unique_cols['ignore_cols'] = ['version', 'update info']
  				rec_list = self.process_table(parent_title, table, page_url, unique_cols)
  
  				# Add the table on the Cadastral page to the CSV
  				for idx, rec in enumerate(rec_list):
  					msg = "Extracting %s of %s records of %s" % (idx + 1, len(rec_list), parent_title)
  					shared.print_oneliner(msg)
  					for k, v in rec.items():
  						pt_csv.add(k, v)
  
  					pt_csv.add('Source', source)
  					pt_csv.write_dataset()
  					
  				print
  
  				# FOR DEBUG ONLY:
  				# print ""
  				# for row in rec_list:
  				#     for k, v in row.items():
  				#         print "%s: %s" % (k, v)
  
  				###########################################################
  				# Next, get the Tabular selection site for scraping
  				tab_a = bsoup.find_tags_containing(rec_soup, 'Tabular selection', 'a')
  				tab_link = tab_a['href']
  				tab_url = urlparse.urljoin(page_url, tab_link)
  
  				# Get the Tabular soup
  				tab_soup = bsoup.get_soup(tab_url)
  				
  				if self.check_result(tab_soup, tab_url, 'Manitoba Land Initiative - %s Sub-Page' % parent_title):
  					ignore_rows = ['Go to top', 'Townships', 'Townships East', 'Townships West',
  								   'Parish', 'DofS', 'Special', 'Projects', 'Portage Diversion',
  								   'Winnipeg Floodway', 'St. Norbert', 'Winnipeg - Index table']
  
  					table_list = tab_soup.find_all('table')
  					unique_cols['mdata_col'] = '.doc'
  					unique_cols['date_col'] = 'date'
  					unique_cols['ref_col'] = 'adjust'
  					unique_cols['ignore_cols'] = ['ver']
  
  					# Go through all the tables on the page (in this case 1)
  					for idx, table in enumerate(table_list):
  						#msg = "Extracting table %s of %s" % (idx + 1, len(table_list))
  						#shared.print_oneliner(msg)
  					
  						rec_list = self.process_table(parent_title, table, tab_url, unique_cols=unique_cols,
  													  ignore_rows=ignore_rows)
  
  						# FOR DEBUG ONLY:
  						# print ""
  						# for row in rec_list:
  						#     for k, v in row.items():
  						#         print "%s: %s" % (k, v)
  
  						for idx, rec in enumerate(rec_list):
  							msg = "Extracting %s of %s records of %s" % (idx + 1, len(rec_list), parent_title)
  							shared.print_oneliner(msg)
  							# Add record to pt_csv
  							for k, v in rec.items():
  								pt_csv.add(k, v)
  
  							# Update the CSV file
  							pt_csv.add('Source', source)
  							pt_csv.write_dataset()
  							
  						print
  
  					###########################################################
  					# There are two items in the list that has a separate page that needs to be loaded.
  					#   St. Norbert and Winnipeg have an item called "Image Map" with a link to another page
  
  					# Get the St. Nobert page URL
  					nobert_a = bsoup.find_tags_containing(tab_soup, 'St. Norbert', 'a')
  					nobert_td = nobert_a.parent
  					imgmap_td = nobert_td.find_next_sibling('td')
  					imgmap_a = imgmap_td.find('a')
  					nobert_link = imgmap_a['href']
  					nobert_url = urlparse.urljoin(tab_url, nobert_link)
  					
  					#print nobert_url
  					#answer = raw_input("Press enter...")
  
  					# Soup the St. Nobert page
  					nobert_soup = bsoup.get_soup(nobert_url)
  					
  					if self.check_result(nobert_soup, nobert_url, 'Manitoba Land Initiative - %s St. Norbert Sub-Page' % parent_title):
  						# Get the areas on the St. Norbert page and soup up there pages
  						nobert_areas = nobert_soup.find_all('area')
  						for idx, area in enumerate(nobert_areas):
  							msg = "Extracting area %s of %s from map" % (idx + 1, len(nobert_areas))
  							shared.print_oneliner(msg)
  						
  							area_link = area['href']
  							area_url = urlparse.urljoin(nobert_url, area_link)
  
  							area_soup = bsoup.get_soup(area_url, True)
  							
  							if area_soup is None:
  								print "\nWARNING: St. Norbert area '%s' could not be opened." % (parent_title, area_url)
  								print "Please check the 'err_log.csv' file in the province/territory results folder."
  								self.write_error(area_url, 'St. Norbert Area' % parent_title, 'Page could not be opened.')
  							else:
  								# Locate the proper table
  								h5 = bsoup.find_tags_containing(area_soup, 'Manitoba Land Initiative', 'h5')
  								table = h5.find_next_sibling('table')
  
  								# Get the title string
  								title_font = bsoup.find_tags_containing(table, 'wred', 'font')
  								if title_font is None: title_font = bsoup.find_tags_containing(table, 'ered', 'font')
  								title_str = "%s - %s" % (parent_title, bsoup.get_text(title_font))
  
  								# Locate the metadata
  								mdata_url = ''
  								mdata_td = bsoup.find_tags_containing(table, 'Metadata', 'td')
  								mdata_tr = mdata_td.parent
  								mdata_download_tr = mdata_tr.find_next_sibling('tr')
  								mdata_a = mdata_download_tr.find('a')
  								if not mdata_a is None:
  									mdata_link = mdata_a['href']
  									mdata_url = urlparse.urljoin(area_url, mdata_link)
  
  								# Locate the downloads
  								downloads = []
  								dwnld_trs = area_soup.find_all('td', text='Download')
  								for dwnload in dwnld_trs:
  									parent_tr = dwnload.parent
  									dwnload_tr = parent_tr.find_next_sibling('tr')
  									dwnload_a = dwnload_tr.find('a')
  									if not dwnload_a is None:
  										dwnload_link = dwnload_a['href']
  										format = dwnload_a.text
  										dwnload_url = urlparse.urljoin(area_url, dwnload_link)
  										downloads.append((format, dwnload_url))
  
  								# Determine the download and access strings
  								if len(downloads) == 0:
  									download_str = 'No'
  									access_str = 'Contact the Province'
  								elif len(downloads) == 1:
  									download_str = downloads[0][1]
  									access_str = 'Download/Web Acccessible'
  								else:
  									download_str = 'Multiple Downloads'
  									access_str = 'Download/Web Acccessible'
  
  								# Get the formats
  								formats = '|'.join(f[0].replace('.', '') for f in downloads)
  
  								# Add all geospatial data to the pt_csv
  								pt_csv.add('Source', source)
  								pt_csv.add('Title', title_str)
  								pt_csv.add('Access', access_str)
  								pt_csv.add('Type', 'Vector Data')
  								pt_csv.add('Download', download_str)
  								pt_csv.add('Web Page URL', area_url)
  								pt_csv.add('Metadata URL', mdata_url)
  								pt_csv.add('Spatial Reference', sp_str)
  								pt_csv.add('Available Formats', formats)
  
  								pt_csv.write_dataset()
  						
  						print
  					# Get the Winnipeg page URL
  					winnipeg_a = bsoup.find_tags_containing(tab_soup, 'Winnipeg - Index table', 'a')
  					winnipeg_td = winnipeg_a.parent
  					imgmap_td = winnipeg_td.find_next_sibling('td')
  					imgmap_a = imgmap_td.find('a')
  					winnipeg_link = imgmap_a['href']
  					winnipeg_url = urlparse.urljoin(tab_url, winnipeg_link)
  
  					# Soup the Winnipeg page
  					winnipeg_soup = bsoup.get_soup(winnipeg_url)
  					
  					if self.check_result(winnipeg_soup, winnipeg_url, 'Manitoba Land Initiative - %s Winnipeg Sub-Page' % parent_title):
  						# Get the areas on the Winnipeg page and soup up there pages
  						winnipeg_areas = winnipeg_soup.find_all('area')
  
  						# Go through all the areas in Winnipeg
  						for idx, area in enumerate(winnipeg_areas):
  							msg = "Extracting area %s of %s from map" % (idx + 1, len(winnipeg_areas))
  							shared.print_oneliner(msg)
  						
  							# Each area in Winnipeg contains a web map page with a collection
  							#   of tiles as <area> tags
  							area_link = area['href']
  							area_url = urlparse.urljoin(winnipeg_url, area_link)
  							# Load the metadata page instead
  							area_url = area_url.replace("shp", "meta")
  
  							area_soup = bsoup.get_soup(area_url, True)
  							
  							if area_soup is None:
  								print "\nWARNING: Winnipeg area '%s' could not be opened." % (parent_title, area_url)
  								print "Please check the 'err_log.csv' file in the province/territory results folder."
  								self.write_error(area_url, 'Winnipeg Area' % parent_title, 'Page could not be opened.')
  							else:
  								# Get the title of the page
  								title_font = area_soup.find('font', attrs={'size': '3'})
  								area_title = title_font.text
  
  								# Collect a list of tiles from the <area> tags
  								tiles = area_soup.find_all('area')
  
  								# Get a list of formats
  								formats = []
  								b_doc = area_soup.find('b', text='.DOC')
  								p_doc = b_doc.parent
  								td_doc = p_doc.parent
  								tr = td_doc.parent
  								td_list = tr.find_all('td')
  								for td in td_list:
  									if not td.text == '.DOC':
  										formats.append(td.text.strip().replace('.', ''))
  
  								# Go through the different areas (tiles)
  								for tile in tiles:
  									# Get the tile name
  									id = tile['alt']
  									# Get the metadata URL
  									mdata_link = tile['href']
  									mdata_url = urlparse.urljoin(area_url, mdata_link)
  
  									title_str = "%s - %s: %s" % (parent_title, area_title, id)
  
  									formats_str = "|".join(formats)
  
  									download_info = shared.get_download_text(formats)
  									download_str, access_str = download_info.split('|')
  
  									# print "Title: "
  									# print title_str
  									# print "Formats: " + str(formats_str)
  									# print "Metadata URL: " + str(mdata_url)
  									# print "Download: " + str(download_str)
  									# print "Access: " + str(access_str)
  
  									# Add all geospatial data to the pt_csv
  									pt_csv.add('Source', source)
  									pt_csv.add('Title', title_str)
  									pt_csv.add('Access', access_str)
  									pt_csv.add('Type', 'Vector Data')
  									pt_csv.add('Download', download_str)
  									pt_csv.add('Web Page URL', area_url)
  									pt_csv.add('Metadata URL', mdata_url)
  									pt_csv.add('Spatial Reference', sp_str)
  									pt_csv.add('Available Formats', formats_str)
  
  									pt_csv.write_dataset()
  
  						print
  				###########################################################
  				# Get the remaining links on the Cadastral page
  
  				# Create a list of the texts found in the links
  				link_texts = ['Cadastral Index', 'Rural', 'Rural_cities', 'Townships',
  							  'Winnipeg', 'All Cadastral Polygons', 'All Cadastral Lines',
  							  'All Cadastral Text']
  				links_table = rec_soup.find_all('table')[0]
  
  				# Cycle through each link
  				for idx, text in enumerate(link_texts):
  					msg = "Extracting %s of %s records" % (idx + 1, len(link_texts))
  					shared.print_oneliner(msg)
  							
  					anchor = bsoup.find_tags_containing(links_table, text, 'a')
  					title_str = "%s - %s" % (parent_title, shared.clean_text(anchor.text))
  					shp_url = shared.get_anchor_url(anchor, page_url)
  
  					# Add all geospatial data to the pt_csv
  					pt_csv.add('Source', source)
  					pt_csv.add('Title', title_str)
  					pt_csv.add('Access', 'Download/Web Accessible')
  					pt_csv.add('Type', 'Vector Data')
  					pt_csv.add('Download', shp_url)
  					pt_csv.add('Web Page URL', page_url)
  					pt_csv.add('Spatial Reference', sp_str)
  					pt_csv.add('Available Formats', 'SHP')
  
  					pt_csv.write_dataset()
  					
  				print
  
  				pt_csv.close_csv()
  
  			elif parent_title == 'Digital Elevation Models':
  
  				# Create the CSV file and determining category
  				csv_fn = "MLI_%s_results" % fn_tag
  				pt_csv = sh.PT_CSV(csv_fn, self)
  				pt_csv.open_csv()
  
  				# Fix the errors on the page
  				html_code = unicode(rec_soup)
  				#print "HTML: %s" % html_code.find('Coverage Overview</font></b></td></tr></tbody></table></br></br></br></br></div></div></div></div></div></body></html>')
  				html_code = html_code.replace("Coverage Overview</font></b></td></tr></tbody></table></br></br></br></br></div></div></div></div></div></body></html>",
  											  "Coverage Overview</font></b></td>")
  				rec_soup = BeautifulSoup(html_code, 'html.parser')
  
  				###########################################################
  				# Start with tables on the Digital Elevation Models page
  
  				table_list = rec_soup.find_all('table')
  
  				# FOR DEBUG ONLY
  				#html_f = codecs.open('html_text.html', encoding='utf_8_sig', mode="w")
  				#html_f.write(unicode(rec_soup))
  				#html_f.close()
  
  				unique_cols = collections.OrderedDict()
  				unique_cols['mdata_col'] = '.doc metadata'
  				replace_vals = [('CONTOURS', 'SHP')]
  				unique_cols['ignore_cols'] = ['version', 'update info']
  				ignore_rows = ['Contours']
  
  				for idx, table in enumerate(table_list):
  					
  					rec_list = self.process_table(parent_title, table, page_url, unique_cols,
  												  ignore_rows=ignore_rows,
  												  replace_vals=replace_vals)
  
  					#print rec_list
  
  					# Add the table on the DEM page to the CSV
  					for idx, rec in enumerate(rec_list):
  						msg = "Extracting %s of %s records" % (idx + 1, len(rec_list))
  						shared.print_oneliner(msg)
  						for k, v in rec.items():
  							pt_csv.add(k, v)
  
  						pt_csv.add('Source', source)
  						pt_csv.write_dataset()
  					print
  
  				###########################################################
  				# Get the Red River sites
  				redriver_font = bsoup.find_tags_containing(rec_soup, 'Red River LiDAR Contours', 'font')
  				parent_td = redriver_font.parent.parent
  				redriver_tds = parent_td.find_next_siblings('td')
  				#print redriver_tds
  				rrmdata_col = redriver_tds[1]
  				rrmdata_a = rrmdata_col.find('a')
  				rrmdata_url = shared.get_anchor_url(rrmdata_a, page_url)
  
  				rr_soup = bsoup.get_soup(rrmdata_url)
  				areas = rr_soup.find_all('area')
  
  				# Get the title of the page
  				title_font = rr_soup.find('font', attrs={'size': '4'})
  				title = title_font.text
  				title = shared.clean_text(title)
  
  				# Get a list of formats
  				formats = []
  				b_doc = rr_soup.find('b', text='.DOC')
  				p_doc = b_doc.parent
  				td_doc = p_doc.parent
  				tr = td_doc.parent
  				td_list = tr.find_all('td')
  				for td in td_list:
  					if not td.text == '.DOC':
  						formats.append(td.text.strip().replace('.', ''))
  
  				for idx, area in enumerate(areas):
  					msg = "Extracting area %s of %s from map" % (idx + 1, len(areas))
  					shared.print_oneliner(msg)
  				
  					# Get the area name
  					id = area['alt']
  					# Get the metadata URL
  					mdata_url = shared.get_anchor_url(area, rrmdata_url)
  
  					title_str = "%s - %s: %s" % (parent_title, title, id)
  
  					formats_str = "|".join(formats)
  
  					download_info = shared.get_download_text(formats)
  					download_str, access_str = download_info.split('|')
  
  					mdata_info = self.get_metadata(mdata_url)
  
  					for k, v in mdata_info.items():
  						pt_csv.add(k, v)
  
  					# Add all geospatial data to the pt_csv
  					pt_csv.add('Source', source)
  					pt_csv.add('Title', title_str)
  					pt_csv.add('Access', access_str)
  					pt_csv.add('Type', 'Vector Data')
  					pt_csv.add('Download', download_str)
  					pt_csv.add('Web Page URL', rrmdata_url)
  					pt_csv.add('Metadata URL', mdata_url)
  					pt_csv.add('Spatial Reference', sp_str)
  					pt_csv.add('Available Formats', formats_str)
  
  					pt_csv.write_dataset()
  					
  				print
  
  				###########################################################
  				# Next get the LIDAR site
  
  				lidar_font = rec_soup.find('font', text='LIDAR(Various')
  				lidar_a = lidar_font.parent
  				lidar_url = shared.get_anchor_url(lidar_a, page_url)
  
  				lidar_soup = bsoup.get_soup(lidar_url)
  
  				tables = lidar_soup.find_all('table', attrs={'border': '2'})
  
  				# Get the first table on the LIDAR page
  				unique_cols['mdata_col'] = 'metadata'
  				unique_cols['ignore_cols'] = ['coverage overview']
  				replace_vals = [('Contours', 'SHP')]
  				table1 = self.process_table(parent_title, tables[0], lidar_url, unique_cols, replace_vals=replace_vals)
  
  				#print table1
  
  				# Add the table on the DEM page to the CSV
  				for idx, row in enumerate(table1):
  					msg = "Extracting %s of %s records" % (idx + 1, len(table1))
  					shared.print_oneliner(msg)
  				
  					for k, v in row.items():
  						pt_csv.add(k, v)
  
  					pt_csv.add('Source', source)
  					pt_csv.write_dataset()
  				
  				print
  
  				# Get the second table on the LIDAR page
  				unique_cols['mdata_col'] = '.doc metadata'
  				unique_cols['ignore_cols'] = ['coverage overview']
  				table1 = self.process_table(parent_title, tables[1], lidar_url, unique_cols, replace_vals=replace_vals)
  
  				#print table1
  
  				# Add the table on the DEM page to the CSV
  				for idx, row in enumerate(table1):
  					msg = "Extracting %s of %s records" % (idx + 1, len(table1))
  					shared.print_oneliner(msg)
  					
  					for k, v in row.items():
  						pt_csv.add(k, v)
  
  					pt_csv.add('Source', source)
  					pt_csv.write_dataset()
  
  				pt_csv.close_csv()
  				
  				print
  
  			elif parent_title == 'Digital Imagery':
  				#print "Digital Imagery"
  
  				# Create the CSV file and determining category
  				csv_fn = "MLI_%s_results" % fn_tag
  				pt_csv = sh.PT_CSV(csv_fn, self)
  				pt_csv.open_csv()
  
  				table = rec_soup.find('table', attrs={'border': '2'})
  
  				rows = table.find_all('tr')
  
  				for idx, row in enumerate(rows):
  					
  					# Get the anchor in this row
  					a = row.a
  					if a is None: continue
  
  					a_text = shared.clean_text(a.text)
  
  					# Get a list of columns from the row
  					cols = row.find_all('td')
  
  					# Get the description from the third column of the row
  					desc_str = shared.clean_text(cols[2].text)
  
  					# Get the metadata from the second column of the row
  					mdata_col = cols[1]
  					mdata_info = None
  					mdata_a = mdata_col.find('a')
  					if mdata_a is not None:
  						mdata_url = shared.get_anchor_url(mdata_a, page_url)
  						mdata_info = self.get_metadata(mdata_url)
  
  					# If the link is not to a page (.html), then skip it
  					sub_url = shared.get_anchor_url(a, page_url)
  					if sub_url.find('.html') == -1: continue
  
  					sub_soup = bsoup.get_soup(sub_url)
  
  					if a_text == 'Capital Region 50cm' or \
  							a_text == 'Nickel Belt' or \
  							a_text == 'Blanshard North' or \
  							a_text == 'Blanshard South' or \
  							a_text == 'North Cypress' or \
  							a_text == 'North Norfolk' or \
  							a_text == 'Park North' or \
  							a_text == 'Upper Assiniboine North' or \
  							a_text == 'Upper Assiniboine South' or \
  							a_text == 'Vermillion River':
  
  						# These pages contain a map with links
  						#   to the datasets
  
  						a_text = a_text.replace(" 50cm", "")
  
  						####################################################################################
  						# Get all the shapefile links on the page
  						if a_text.find('Blanshard') > -1:
  							shp_titles = ['%s ortho grid in SHAPE format' % a_text,
  								  'Blanshard (North & South - all)']
  						else:
  							shp_titles = ['%s ortho grid in SHAPE format' % a_text]
  						down_res = bsoup.get_link_datasets(sub_soup, shp_titles, sub_url, 'a')
  
  						for idx, res in enumerate(down_res):
  							msg = "Extracting %s of %s records in %s" % (idx + 1, len(down_res), a_text)
  							shared.print_oneliner(msg)
  						
  							for k, v in res.items():
  								pt_csv.add(k, v)
  
  							# Add the metadata info to the CSV file
  							if mdata_info is not None:
  								for k, v in mdata_info.items():
  									pt_csv.add(k, v)
  
  							if res['Available Formats'] == 'SID' or \
  											res['Available Formats'] == 'TIF':
  								dtype = 'Raster Data'
  							else:
  								dtype = 'Vector Data'
  
  							# Add all geospatial data to the pt_csv
  							pt_csv.add('Source', source)
  							pt_csv.add('Type', dtype)
  							pt_csv.add('Web Page URL', sub_url)
  							pt_csv.add('Metadata URL', mdata_url)
  							pt_csv.add('Description', desc_str)
  							pt_csv.add('Spatial Reference', sp_str)
  
  							pt_csv.write_dataset()
  
  						print
  						############################################################################
  						# Get the map links
  
  						area_map = sub_soup.find('map')
  
  						# Get the world Zip file for the datasets
  						world_txt = '%s Ortho World files (.SDW)' % a_text
  						world_url, notes_str = self.get_world_link(sub_soup, sub_url, world_txt)
  
  						# Get the map info on the sub page
  						area_map_info = self.process_map(area_map, sub_url, a_text)
  
  						# Add the map info to the CSV file
  						for idx, ds in enumerate(area_map_info):
  							msg = "Extracting %s of %s areas on map" % (idx + 1, len(area_map_info))
  							shared.print_oneliner(msg)
  						
  							for k, v in ds.items():
  								pt_csv.add(k, v)
  
  							# Add the metadata info to the CSV file
  							if mdata_info is not None:
  								for k, v in mdata_info.items():
  									pt_csv.add(k, v)
  
  							pt_csv.add('Source', source)
  							pt_csv.add('Available Formats', 'SID')
  							pt_csv.add('Type', 'Raster Data')
  							pt_csv.add('Description', desc_str)
  							pt_csv.add('Data URL', world_url)
  							pt_csv.add('Metadata URL', mdata_url)
  							pt_csv.add('Spatial Reference', sp_str)
  							pt_csv.add('Notes', notes_str)
  
  							pt_csv.write_dataset()
  							
  						print
  
  					elif a_text == '1Km Tiles':
  						# Get the 1km tiles
  
  						#print "1km tiles"
  
  						#########################################################
  						# First, process the map on the 1km tiles page
  						idx_map = sub_soup.find('map')
  						map_info = self.process_map(idx_map, sub_url)
  
  						for idx, area in enumerate(map_info):
  							# Each link on the map goes to another page with a
  							#   map
  
  							# Get the download URL which is actually the sub page
  							area_url = area['Download']
  							area_soup = bsoup.get_soup(area_url)
  							area_map = area_soup.find('map')
  
  							#####################################################
  							# Get all the image datasets from the map on the
  							#   sub page
  
  							# Get the world Zip file for the datasets
  							world_txt = 'World Files required'
  							world_url, notes_str = self.get_world_link(sub_soup, sub_url, world_txt)
  
  							# Get the map info on the sub page
  							area_map_info = self.process_map(area_map, area['Download'], '1km Ortho')
  
  							# Add the map info to the CSV file
  							for idx, ds in enumerate(area_map_info):
  								msg = "Extracting %s of %s areas in %s" % (idx + 1, len(area_map_info), a_text)
  								shared.print_oneliner(msg)
  							
  								for k, v in ds.items():
  									pt_csv.add(k, v)
  
  								if mdata_info is not None:
  									for k, v in mdata_info.items():
  										pt_csv.add(k, v)
  
  								pt_csv.add('Source', source)
  								pt_csv.add('Available Formats', 'TIF')
  								pt_csv.add('Type', 'Raster Data')
  								pt_csv.add('Data URL', world_url)
  								pt_csv.add('Description', desc_str)
  								pt_csv.add('Metadata URL', mdata_url)
  								pt_csv.add('Spatial Reference', sp_str)
  								pt_csv.add('Notes', notes_str)
  
  								pt_csv.write_dataset()
  								
  							print
  
  					elif a_text == '5Km Tiles':
  
  						#########################################################
  						# Get all the shapefile links on the page
  						shp_titles = ['Download 5K Ortho grid in SHAPE format',
  									  'Download Township Grid in SHAPE format']
  						down_res = bsoup.get_link_datasets(sub_soup, shp_titles, sub_url)
  
  						for idx, res in enumerate(down_res):
  							msg = "Extracting %s of %s records in %s" % (idx + 1, len(down_res), a_text)
  							shared.print_oneliner(msg)
  						
  							for k, v in res.items():
  								pt_csv.add(k, v)
  
  							# Add all geospatial data to the pt_csv
  							pt_csv.add('Source', source)
  							pt_csv.add('Type', 'Vector Data')
  							pt_csv.add('Web Page URL', sub_url)
  							pt_csv.add('Description', desc_str)
  							pt_csv.add('Spatial Reference', sp_str)
  
  							pt_csv.write_dataset()
  						
  						print
  
  						# Get the world Zip file for the datasets
  						world_txt = 'Download "world files" - (.tfw)'
  						world_url, notes_str = self.get_world_link(sub_soup, sub_url, world_txt)
  
  						#########################################################
  						# Process the map on the 5km tiles page
  						idx_map = sub_soup.find('map')
  						map_info = self.process_map(idx_map, sub_url)
  
  						for area in map_info:
  							# Each link on the map goes to another page with a
  							#   map
  
  							# Get the download URL which is actually the sub page
  							sub_url = area['Download']
  							if sub_url.find('.html') > -1:
  								#####################################################
  								# Get all the image datasets from the map on the
  								#   sub page
  
  								area_soup = bsoup.get_soup(sub_url)
  								area_map = area_soup.find('map')
  
  								# Get the map info on the sub page
  								area_map_info = self.process_map(area_map, area['Download'], '5km Ortho')
  
  								# Add the map info to the CSV file
  								for idx, ds in enumerate(area_map_info):
  									msg = "Extracting %s of %s areas in %s" % (idx + 1, len(area_map_info), a_text)
  									shared.print_oneliner(msg)
  								
  									for k, v in ds.items():
  										pt_csv.add(k, v)
  
  									if mdata_info is not None:
  										for k, v in mdata_info.items():
  											pt_csv.add(k, v)
  
  									pt_csv.add('Source', source)
  									pt_csv.add('Available Formats', 'TIF')
  									pt_csv.add('Type', 'Raster Data')
  									pt_csv.add('Data URL', world_url)
  									pt_csv.add('Description', desc_str)
  									pt_csv.add('Metadata URL', mdata_url)
  									pt_csv.add('Spatial Reference', sp_str)
  									pt_csv.add('Notes', notes_str)
  
  									pt_csv.write_dataset()
  									
  								print
  
  					elif a_text == 'Mosaics - MrSid':
  						#########################################################
  						# Get all the shapefile links on the page
  
  						shp_titles = ['Download MrSid Ortho grid in SHAPE format',
  									  'Download 5K Ortho grid in SHAPE format',
  									  'Download Township grid in SHAPE format']
  						down_res = bsoup.get_link_datasets(sub_soup, shp_titles, sub_url)
  
  						for idx, res in enumerate(down_res):
  							msg = "Extracting %s of %s records in %s" % (idx + 1, len(down_res), a_text)
  							shared.print_oneliner(msg)
  						
  							for k, v in res.items():
  								pt_csv.add(k, v)
  
  							# Add all geospatial data to the pt_csv
  							pt_csv.add('Source', source)
  							pt_csv.add('Type', 'Vector Data')
  							pt_csv.add('Web Page URL', sub_url)
  							pt_csv.add('Description', desc_str)
  							pt_csv.add('Spatial Reference', sp_str)
  
  							pt_csv.write_dataset()
  						
  						print
  
  						#########################################################
  						# Now get all the image datasets from the map on the page
  
  						# Get the world Zip file for the datasets
  						world_txt = 'world files'
  						world_url, notes_str = self.get_world_link(sub_soup, sub_url, world_txt)
  
  						area_map = sub_soup.find('map')
  
  						# Get the map info on the sub page
  						area_map_info = self.process_map(area_map, sub_url, 'MrSid  Mosaic Ortho')
  
  						# Add the map info to the CSV file
  						for idx, ds in enumerate(area_map_info):
  							msg = "Extracting %s of %s areas in %s" % (idx + 1, len(area_map_info), a_text)
  							shared.print_oneliner(msg)
  						
  							for k, v in ds.items():
  								pt_csv.add(k, v)
  
  							# Add the metadata info to the CSV file
  							if mdata_info is not None:
  								for k, v in mdata_info.items():
  									pt_csv.add(k, v)
  
  							pt_csv.add('Source', source)
  							pt_csv.add('Available Formats', 'MrSID')
  							pt_csv.add('Type', 'Raster Data')
  							pt_csv.add('Spatial Reference', sp_str)
  							pt_csv.add('Metadata URL', mdata_url)
  							pt_csv.add('Data URL', world_url)
  							pt_csv.add('Notes', notes_str)
  
  							pt_csv.write_dataset()
  							
  						print
  
  					elif a_text == 'Nickel Belt - MrSid':
  						print "\nExtracting 1 record in %s" % a_text
  					
  						# Get the world Zip file for the datasets
  						world_txt = '.SDW'
  						world_url, notes_str = self.get_world_link(sub_soup, sub_url, world_txt)
  
  						table = sub_soup.find('table')
  
  						ignore_rows = ['Nickel Belt World File - All']
  						table_rows = self.process_table(parent_title, table, sub_url, ignore_rows=ignore_rows)
  
  						for res in table_rows:
  							for k, v in res.items():
  								pt_csv.add(k, v)
  
  						# Add the metadata info to the CSV file
  						if mdata_info is not None:
  							for k, v in mdata_info.items():
  								pt_csv.add(k, v)
  
  						pt_csv.add('Source', source)
  						pt_csv.add('Description', desc_str)
  						pt_csv.add('Spatial Reference', sp_str)
  						pt_csv.add('Data URL', world_url)
  						pt_csv.add('Metadata URL', mdata_url)
  						pt_csv.add('Notes', notes_str)
  
  						pt_csv.write_dataset()
  
  					elif a_text == 'IRS':
  						print "\nExtracting 1 record in %s" % a_text
  					
  						# Get the world Zip file for the datasets
  						world_txt = '.SDW'
  						world_url, notes_str = self.get_world_link(sub_soup, sub_url, world_txt)
  
  						table = sub_soup.find('table', attrs={'border': '1'})
  
  						pt_csv.add('Description', desc_str)
  
  						unique_cols = collections.OrderedDict()
  						unique_cols['mdata_col'] = '.doc'
  						unique_cols['ignore_cols'] = ['.gif', '.sdw']
  						table_rows = self.process_table(parent_title, table, sub_url, unique_cols)
  
  						for res in table_rows:
  							for k, v in res.items():
  								pt_csv.add(k, v)
  
  						# Add the metadata info to the CSV file
  						if mdata_info is not None:
  							for k, v in mdata_info.items():
  								pt_csv.add(k, v)
  
  						pt_csv.add('Source', source)
  						pt_csv.add('Spatial Reference', sp_str)
  						pt_csv.add('Data URL', world_url)
  						pt_csv.add('Metadata URL', mdata_url)
  						pt_csv.add('Notes', notes_str)
  
  						pt_csv.write_dataset()
  
  					elif a_text == 'LandSat_7':
  						# The LandSat_7 page contains a map with links
  						#   to other pages containing download links
  
  						# Get all the shapefile links on the page
  						shp_titles = ['Download LandSat7 WRS grid in SHAPE format',
  									  'Download NTS grid in SHAPE format',
  									  'Download 5K Ortho grid in SHAPE format',
  									  'Download Township Grid in SHAPE format']
  						down_res = bsoup.get_link_datasets(sub_soup, shp_titles, sub_url)
  
  						for idx, res in enumerate(down_res):
  							msg = "Extracting %s of %s records in %s" % (idx + 1, len(down_res), a_text)
  							shared.print_oneliner(msg)
  						
  							for k, v in res.items():
  								pt_csv.add(k, v)
  
  							# Add all geospatial data to the pt_csv
  							pt_csv.add('Source', source)
  							pt_csv.add('Type', 'Vector Data')
  							pt_csv.add('Web Page URL', sub_url)
  							pt_csv.add('Description', desc_str)
  							pt_csv.add('Spatial Reference', sp_str)
  
  							pt_csv.write_dataset()
  							
  						print
  
  						# Process the map on the page
  						idx_map = sub_soup.find('map')
  						map_info = self.process_map(idx_map, sub_url)
  
  						for idx, area in enumerate(map_info):
  						
  							# Get the download URL which is actually the sub page
  							area_url = area['Download']
  							if area_url.find('.html') > -1:
  								area_soup = bsoup.get_soup(area_url, True)
  
  								# Get the world Zip file for the datasets
  								world_txt = '.SDW'
  								world_url, notes_str = self.get_world_link(sub_soup, area_url, world_txt)
  
  								# Locate a column which is green
  								green_td = area_soup.find('td', attrs={'bgcolor': '#00FF00'})
  								# Go to the parent row
  								parent_tr = green_td.parent
  								tbody = parent_tr.parent
  
  								# Get all the rows
  								rows = tbody.find_all('tr')
  								#for row in rows:
  								#	print ""
  								#	print row
  								#print "Number of rows: " + str(len(rows))
  
  								title_row = rows[1].find_all('td')
  
  								mdata_col = rows[2].find('td')
  								#print mdata_row
  								mdata_a = mdata_col.a
  								mdata_url = shared.get_anchor_url(mdata_a, area_url)
  
  								sp_col = rows[3].find('td')
  								sp_a = sp_col.a
  								sp_str = shared.get_anchor_url(sp_a, area_url)
  
  								# Cycle through each column in the table
  								for index, col in enumerate(title_row[1:]):
  									msg = "Extracting %s of %s images in %s" % (index + 1, len(title_row[1:]), a_text)
  									shared.print_oneliner(msg)
  								
  									# Get the title from the first row
  									title = shared.clean_text(col.text)
  									title_str = '%s - Band %s' % (parent_title, title)
  
  									# Get the formats from the second and third row
  									formats = []
  									for row in rows[2:3]:
  										cols = row.find_all('td')
  										#print index
  										#print "Number of columns: " + str(len(cols))
  										format = shared.clean_text(cols[index + 1].text)
  										format = format.replace('.', '').upper()
  										download_a = cols[index + 1].a
  										download_url = shared.get_anchor_url(download_a, area_url)
  										if not format == '':
  											formats.append(format)
  
  									download_info = shared.get_download_text(formats, download_url)
  									download_str, access_str = download_info.split('|')
  
  									# print "Title: " + str(title_str)
  									# print "Formats: " + str("|".join(formats))
  									# print "Spatial Reference: " + str(sp_str)
  									# print "Description: " + str(desc_str)
  
  									# Add the metadata info to the CSV file
  									if mdata_info is not None:
  										for k, v in mdata_info.items():
  											pt_csv.add(k, v)
  
  									# Add all geospatial data to the pt_csv
  									pt_csv.add('Source', source)
  									pt_csv.add('Title', title_str)
  									pt_csv.add('Access', access_str)
  									pt_csv.add('Type', 'Raster Data')
  									pt_csv.add('Download', download_str)
  									pt_csv.add('Web Page URL', sub_url)
  									pt_csv.add('Metadata URL', mdata_url)
  									pt_csv.add('Spatial Reference', sp_str)
  									pt_csv.add('Available Formats', "|".join(formats))
  									pt_csv.add('Data URL', world_url)
  									pt_csv.add('Notes', notes_str)
  
  									pt_csv.write_dataset()
  						
  						print
  					elif a_text == 'Lake_St_Martin_part1' or a_text == 'Lake_St_Martin_part2':
  
  						print "\nExtracting 1 record in %s" % a_text
  					
  						formats = ['SID']
  						download_info = shared.get_download_text(sub_url)
  						download_str, access_str = download_info.split('|')
  
  						# Add the metadata info to the CSV file
  						if mdata_info is not None:
  							for k, v in mdata_info.items():
  								pt_csv.add(k, v)
  
  						# Add all geospatial data to the pt_csv
  						pt_csv.add('Source', source)
  						pt_csv.add('Title', title_str)
  						pt_csv.add('Access', access_str)
  						pt_csv.add('Type', 'Raster Data')
  						pt_csv.add('Download', download_str)
  						pt_csv.add('Web Page URL', page_url)
  						pt_csv.add('Metadata URL', mdata_url)
  						pt_csv.add('Spatial Reference', sp_str)
  						pt_csv.add('Available Formats', "|".join(formats))
  
  						pt_csv.write_dataset()
  
  					elif a_text == 'Refresh - (2007-2014) Color':
  
  						#########################################################
  						# Get all the shapefile links on the page
  
  						shp_titles = ['Ortho refresh Years 1&2 (2007-2009) 1m pixel MrSid Color 5Km tiles',
  									  'Ortho refresh Years 3&4 (2008-2010) 1m pixel MrSid Color 5Km tiles',
  									  'Refresh 2007-2014 Flight Lines (.SHP)',
  									  'Ortho refresh Year 7 Eastern Block NE_N 1m MrSID Mosaic',
  									  'Ortho refresh Year 7 Eastern Block NE_S 1m MrSID Mosaic',
  									  'Ortho refresh Year 7 Eastern Block SE 1m MrSID Mosaic']
  						down_res = bsoup.get_link_datasets(sub_soup, shp_titles, sub_url, 'a')
  
  						for idx, res in enumerate(down_res):
  							msg = "Extracting %s of %s records in %s" % (idx + 1, len(down_res), a_text)
  							shared.print_oneliner(msg)
  						
  							for k, v in res.items():
  								pt_csv.add(k, v)
  
  							if res['Available Formats'] == 'SID' or \
  									res['Available Formats'] == 'TIF':
  								dtype = 'Raster Data'
  							else:
  								dtype = 'Vector Data'
  
  							# Add the metadata info to the CSV file
  							if mdata_info is not None:
  								for k, v in mdata_info.items():
  									pt_csv.add(k, v)
  
  							# Add all geospatial data to the pt_csv
  							pt_csv.add('Source', source)
  							pt_csv.add('Type', dtype)
  							pt_csv.add('Web Page URL', sub_url)
  							pt_csv.add('Description', desc_str)
  							pt_csv.add('Metadata URL', mdata_url)
  							pt_csv.add('Spatial Reference', sp_str)
  
  							pt_csv.write_dataset()
  							
  						print
  
  						# Find the <tr> with a certain text
  						tr = bsoup.find_tags_containing(sub_soup, 'MrSid color Mosaic tiles',
  													   'tr')
  
  						title_str = ''
  						counter = 0
  						while title_str.find('se  1.0m') == -1:
  							counter += 1
  							msg = "Extracting %s in %s" % (counter, a_text)
  							shared.print_oneliner(msg)
  						
  							# Divide the row into the columns
  							cols = tr.find_all('td')
  
  							# Get the title from the second column
  							title_str = "%s - %s" % (parent_title, bsoup.get_text(cols[1]))
  
  							# Get the world file from the third column
  							world_a = cols[2].a
  							world_url = shared.get_anchor_url(world_a, sub_url)
  							notes_str = "The 'Data URL' contains a link to a Zip file which " \
  										"contains the World file for this dataset."
  
  							# Add the metadata info to the CSV file
  							if mdata_info is not None:
  								for k, v in mdata_info.items():
  									pt_csv.add(k, v)
  
  							pt_csv.add('Source', source)
  							pt_csv.add('Title', title_str)
  							pt_csv.add('Available Formats', 'SID')
  							pt_csv.add('Type', 'Raster Data')
  							pt_csv.add('Description', desc_str)
  							pt_csv.add('Data URL', world_url)
  							pt_csv.add('Metadata URL', mdata_url)
  							pt_csv.add('Spatial Reference', sp_str)
  							pt_csv.add('Notes', notes_str)
  
  							pt_csv.write_dataset()
  
  							tr = tr.find_next_sibling('tr')
  
  							#print title_str
  							#print title_str.find('se  1.0m')
  							
  						print
  
  					elif a_text == 'Upper Assiniboine River':
  						table = sub_soup.find('table', attrs={'cellspacing': '1'})
  
  						a_list = table.find_all('a')
  						for idx, a in enumerate(a_list):
  							msg = "Extracting %s of %s records in %s" % (idx + 1, len(a_list), a_text)
  							shared.print_oneliner(msg)
  						
  							title_str = "%s - %s" % (parent_title, shared.clean_text(a.text))
  							download_url = shared.get_anchor_url(a, sub_url)
  							formats = ['SHP']
  							
  							download_info = shared.get_download_text(formats, download_url)
  							download_str, access_str = download_info.split('|')
  
  							# Add all geospatial data to the pt_csv
  							pt_csv.add('Title', title_str)
  							pt_csv.add('Access', access_str)
  							pt_csv.add('Type', 'Raster Data')
  							pt_csv.add('Download', download_str)
  							pt_csv.add('Web Page URL', page_url)
  							pt_csv.add('Metadata URL', mdata_url)
  							pt_csv.add('Spatial Reference', sp_str)
  							pt_csv.add('Available Formats', "|".join(formats))
  							
  						print
  
  					elif a_text == '5K Ortho grid in .SHP format' or \
  							a_text == '5K Ortho grid in KMZ format' or \
  							a_text == 'Township Grid in SHP format':
  							
  						print "\nExtracting 1 record in %s" % a_text
  
  						if sub_url.find('5K Ortho grid') > -1:
  							title_str = '%s - 5K Ortho grid' % parent_title
  							formats = ['SHP', 'KMZ']
  						else:
  							title_str = '%s - Township Grid' % parent_title
  							formats = ['SHP']
  
  						download_info = shared.get_download_text(formats, sub_url)
  						download_str, access_str = download_info.split('|')
  
  						# Add the metadata info to the CSV file
  						if mdata_info is not None:
  							for k, v in mdata_info.items():
  								pt_csv.add(k, v)
  
  						# Add all geospatial data to the pt_csv
  						pt_csv.add('Source', source)
  						pt_csv.add('Title', title_str)
  						pt_csv.add('Access', access_str)
  						pt_csv.add('Type', 'Raster Data')
  						pt_csv.add('Download', download_str)
  						pt_csv.add('Web Page URL', page_url)
  						pt_csv.add('Metadata URL', mdata_url)
  						pt_csv.add('Spatial Reference', sp_str)
  						pt_csv.add('Available Formats', "|".join(formats))
  
  						pt_csv.write_dataset()
  
  				print
  			elif parent_title == 'Forest Inventory':
  				#print 'Forest Inventory'
  
  				# Create the CSV file and determining category
  				csv_fn = "MLI_%s_results" % fn_tag
  				pt_csv = sh.PT_CSV(csv_fn, self)
  				pt_csv.open_csv()
  
  				table_list = rec_soup.find_all('table')
  
  				#################################################################
  				# Get the fires table
  				fires_table = table_list[0]
  				unique_cols = collections.OrderedDict()
  				unique_cols['mdata_col'] = '.doc'
  				unique_cols['ignore_cols'] = ['.gif', 'update info']
  				ignore_rows = ['Boundaries']
  				replace_vals = [('.GEO', 'FGDB')]
  				rec_list = self.process_table(parent_title, fires_table, page_url, unique_cols,
  											  ignore_rows=ignore_rows, replace_vals=replace_vals)
  
  				#print rec_list
  				#answer = raw_input("Press enter...")
  
  				# Add the table on the Forest Inventory page to the CSV
  				for idx, rec in enumerate(rec_list):
  					msg = "Extracting %s of %s records in %s" % (idx + 1, len(rec_list), parent_title)
  					shared.print_oneliner(msg)
  				
  					for k, v in rec.items():
  						pt_csv.add(k, v)
  
  					pt_csv.add('Source', source)
  					pt_csv.write_dataset()
  				
  				print
  
  				#################################################################
  				# Get the inventory table
  				inv_table = table_list[1]
  
  				# The table has to be divided into two by its blank middle column
  				#   The table_to_dict will append a number to any duplicate column names
  				inv_rows = shared.table_to_dict(inv_table, 0, start_row=1)
  				# Create the lists which will store the two halves of the table
  				first_half = []
  				sec_half = []
  				for row in inv_rows:
  					# Create a list of tuples containing only columns with a '1'
  					first_cols = [(key[:len(key)-1], value) for key, value in row.items() if '1' in key.lower()]
  					# Convert the list of tuples to a dictionary
  					first_cols_dict = dict((x, y) for x, y in first_cols)
  					# Add the dictionary to the first_half list of rows
  					first_half.append(first_cols_dict)
  					# Create a list of tuples containing only columns with a '2'
  					sec_cols = [(key[:len(key)-1], value) for key, value in row.items() if '2' in key.lower()]
  					# Convert the list of tuples to a dictionary
  					sec_cols_dict = dict((x, y) for x, y in sec_cols)
  					# Add the dictionary to the sec_half list of rows
  					sec_half.append(sec_cols_dict)
  
  				all_rows = first_half + sec_half
  
  				for idx, row in enumerate(all_rows):
  					msg = "Extracting dataset %s of %s" % (idx + 1, len(all_rows))
  					shared.print_oneliner(msg)
  				
  					# Get the title
  					title_str = "%s - %s" % (parent_title, shared.clean_text(row['fmu'].text))
  					if not title_str == '':
  						# Get the date
  						date_str = shared.clean_text(row['date'].text)
  
  						# Get the metadata URL
  						mdata = row['.doc']
  						mdata_a = mdata.find('a')
  						mdata_url = shared.get_anchor_url(mdata_a, page_url)
  
  						# If the metadata is an HTML
  						if mdata_url.find('.html') > -1:
  							mdata_info = self.get_metadata(mdata_url)
  
  							for k, v in mdata_info.items():
  								pt_csv.add(k, v)
  
  						# Get the formats and downloads
  						formats = []
  						downloads = []
  						# Get the shapefile if applicable
  						shp_format = shared.clean_text(row['.shp'].text)
  						download_a = row['.shp'].find('a')
  						download = shared.get_anchor_url(download_a, page_url)
  						if not shp_format == '':
  							formats.append('SHP')
  							downloads.append(download)
  						# Get the GEO if applicable
  						geo_format = shared.clean_text(row['.geo'].text)
  						download_a = row['.geo'].find('a')
  						download = shared.get_anchor_url(download_a, page_url)
  						if not geo_format == '':
  							formats.append('FGDB')
  							downloads.append(download)
  
  						download_info = shared.get_download_text(formats, downloads)
  						download_str, access_str = download_info.split('|')
  
  						# print ""
  						# print "Title: " + str(title_str)
  						# print "Date: " + str(date_str)
  						# print "Formats: " + str('|'.join(formats))
  						# print "Metadata URL: " + str(mdata_url)
  
  						# Add all geospatial data to the pt_csv
  						pt_csv.add('Source', source)
  						pt_csv.add('Title', title_str)
  						pt_csv.add('Access', access_str)
  						pt_csv.add('Type', 'Vector Data')
  						pt_csv.add('Download', download_str)
  						pt_csv.add('Web Page URL', page_url)
  						pt_csv.add('Metadata URL', mdata_url)
  						pt_csv.add('Available Formats', '|'.join(formats))
  
  						pt_csv.write_dataset()
  
  				print
  				#################################################################
  				# Get the shapefile downloads at the bottom of page
  
  				# Get the RFQ shapefile
  				rfq_a = rec_soup.find('a', text='RFQ_shp')
  				rfq_url = shared.get_anchor_url(rfq_a, page_url)
  
  				title_str = "%s - %s" % (parent_title, shared.clean_text(rfq_a.text))
  				access_str = 'Download/Web Accessible'
  				download_str = rfq_url
  
  				# Add all geospatial data to the pt_csv
  				pt_csv.add('Source', source)
  				pt_csv.add('Title', title_str)
  				pt_csv.add('Access', access_str)
  				pt_csv.add('Type', 'Vector Data')
  				pt_csv.add('Download', download_str)
  				pt_csv.add('Web Page URL', page_url)
  				pt_csv.add('Available Formats', 'SHP')
  
  				pt_csv.write_dataset()
  
  				# Get the FRI lines
  				fri_a = bsoup.find_tags_containing(rec_soup, 'FRI Lines', 'a')
  				fri_url = shared.get_anchor_url(fri_a, page_url)
  
  				title_str = "%s - %s" % (parent_title, shared.clean_text(fri_a.text))
  				access_str = 'Download/Web Accessible'
  				download_str = fri_url
  
  				# Add all geospatial data to the pt_csv
  				pt_csv.add('Source', source)
  				pt_csv.add('Title', title_str)
  				pt_csv.add('Access', access_str)
  				pt_csv.add('Type', 'Vector Data')
  				pt_csv.add('Download', download_str)
  				pt_csv.add('Web Page URL', page_url)
  				pt_csv.add('Available Formats', 'SHP')
  
  				pt_csv.write_dataset()
  
  			elif parent_title == 'Geology Mapping':
  				#print 'Geology Mapping'
  
  				# Create the CSV file and determining category
  				csv_fn = "MLI_%s_results" % fn_tag
  				pt_csv = sh.PT_CSV(csv_fn, self)
  				pt_csv.open_csv()
  
  				# Get the link to the 1:1,000,000 table
  				million_a = bsoup.find_tags_containing(rec_soup, '1: 1,000,000 Geology', 'a')
  				million_url = shared.get_anchor_url(million_a, page_url)
  
  				million_soup = bsoup.get_soup(million_url)
  
  				table = million_soup.find('table', attrs={'border': '2'})
  
  				unique_cols = collections.OrderedDict()
  				unique_cols['mdata_col'] = '.doc metadata'
  				unique_cols['ignore_cols'] = ['.gif overview', '.tif download']
  				rec_list = self.process_table(parent_title, table, page_url, unique_cols)
  
  				# Add the table on the Forest Inventory page to the CSV
  				for idx, rec in enumerate(rec_list):
  					msg = "Extracting %s of %s records" % (idx + 1, len(rec_list))
  					shared.print_oneliner(msg)
  				
  					for k, v in rec.items():
  						pt_csv.add(k, v)
  
  					pt_csv.add('Source', source)
  					pt_csv.write_dataset()
  					
  				print
  
  			elif parent_title == 'Hydrography':
  				#print 'Hydrology'
  
  				# Create the CSV file and determining category
  				csv_fn = "MLI_%s_results" % fn_tag
  				pt_csv = sh.PT_CSV(csv_fn, self)
  				pt_csv.open_csv()
  
  				###################################################################
  				# Start with the Designated Drain Watercourses
  
  				# Get the Designated Drain Watercourses link
  				drain_b = rec_soup.find('b', text='Designated Drain Watercourses')
  				drain_a = drain_b.find_next_sibling('a')
  				drain_url = shared.get_anchor_url(drain_a, page_url)
  
  				# Get the soup
  				drain_soup = bsoup.get_soup(drain_url)
  
  				# Get the 2 tables on the drain page
  				tables = drain_soup.find_all('table', attrs={'border': '2'})
  
  				# Set the unique_cols
  				unique_cols = collections.OrderedDict()
  				unique_cols['mdata_col'] = '.doc'
  				unique_cols['ignore_cols'] = ['.gif']
  				for idx, table in enumerate(tables):
  					msg = "Extracting table %s of %s" % (idx + 1, len(tables))
  					shared.print_oneliner(msg)
  				
  					rec_list = self.process_table(parent_title, table, page_url, unique_cols)
  
  					# Add the table of the Designated Drain Watercourses page to the CSV
  					for rec in rec_list:
  						for k, v in rec.items():
  							pt_csv.add(k, v)
  
  						pt_csv.add('Source', source)
  						pt_csv.write_dataset()
  				print
  
  				###################################################################
  				# Next, the Basins & Watersheds of Manitoba
  
  				# Get the Basins & Watersheds of Manitoba link
  				basins_b = rec_soup.find('b', text='Basins & Watersheds of Manitoba')
  				basins_a = basins_b.find_next_sibling('a')
  				basins_url = shared.get_anchor_url(basins_a, page_url)
  
  				# Get the soup
  				basins_soup = bsoup.get_soup(basins_url)
  
  				# Get the 2 tables on the drain page
  				tables = basins_soup.find_all('table', attrs={'border': '2'})
  
  				# Set the unique_cols
  				unique_cols = collections.OrderedDict()
  				unique_cols['mdata_col'] = '.doc'
  				unique_cols['ignore_cols'] = ['.gif']
  				for idx, table in enumerate(tables):
  					msg = "Extracting table %s of %s" % (idx + 1, len(tables))
  					shared.print_oneliner(msg)
  					
  					rec_list = self.process_table(parent_title, table, page_url, unique_cols)
  
  					# Add the table of the Basins & Watersheds of Manitoba page to the CSV
  					for rec in rec_list:
  						for k, v in rec.items():
  							pt_csv.add(k, v)
  
  						pt_csv.add('Source', source)
  						pt_csv.write_dataset()
  				print
  
  				###################################################################
  				# The Floods page
  
  				# Get the Floods link
  				floods_a = rec_soup.find('a', text='Floods')
  				floods_url = shared.get_anchor_url(floods_a, page_url)
  
  				floods_soup = bsoup.get_soup(floods_url)
  
  				# Get the Red River flood table
  				redriver_table = floods_soup.find('table', attrs={'id': 'table1'})
  				# Set the unique_cols
  				unique_cols = collections.OrderedDict()
  				unique_cols['mdata_col'] = '.doc'
  				unique_cols['ignore_cols'] = ['.gif']
  				title_prefix = 'Red River Floods'
  				rr_list = self.process_table(parent_title, redriver_table, page_url, unique_cols,
  											 title_prefix=title_prefix)
  
  				# Get the Assiniboine River flood table
  				assiniboine_table = floods_soup.find('table', attrs={'id': 'table2'})
  				# Set the unique_cols
  				unique_cols = collections.OrderedDict()
  				unique_cols['mdata_col'] = '.doc'
  				unique_cols['ignore_cols'] = ['.gif']
  				title_prefix = 'Assiniboine River Floods'
  				assiniboine_list = self.process_table(parent_title, assiniboine_table, page_url,
  															unique_cols,
  															title_prefix=title_prefix)
  
  				# Combine both table results
  				rec_list = rr_list + assiniboine_list
  
  				# Add the table of the Red River floods page to the CSV
  				for idx, rec in enumerate(rec_list):
  					msg = "Extracting %s of %s records in %s" % (idx + 1, len(rec_list), parent_title)
  					shared.print_oneliner(msg)
  					
  					for k, v in rec.items():
  						pt_csv.add(k, v)
  
  					pt_csv.add('Source', source)
  					pt_csv.write_dataset()
  					
  				print
  
  				###################################################################
  				# The Seamless Water page
  
  				# Get the Seamless Water link
  				seam_a = bsoup.find_tags_containing(rec_soup, 'Seamless Water', 'a')
  				seam_url = shared.get_anchor_url(seam_a, page_url)
  
  				seam_soup = bsoup.get_soup(seam_url)
  
  				# Get the Seamless Water table
  				table = seam_soup.find('table', attrs={'id': 'table2'})
  				# Set the unique_cols
  				unique_cols = collections.OrderedDict()
  				unique_cols['mdata_col'] = '.doc'
  				unique_cols['ignore_cols'] = ['.gif']
  				rec_list = self.process_table(parent_title, table, page_url, unique_cols)
  
  				# Add the table of the Seamless Water page to the CSV
  				for idx, rec in enumerate(rec_list):
  					msg = "Extracting %s of %s records" % (idx + 1, len(rec_list))
  					shared.print_oneliner(msg)
  				
  					for k, v in rec.items():
  						pt_csv.add(k, v)
  
  					pt_csv.add('Source', source)
  					pt_csv.write_dataset()
  					
  				print
  
  			elif parent_title == 'Municipal Maps':
  				#print 'Municipal Maps'
  
  				# Create the CSV file and determining category
  				csv_fn = "MLI_%s_results" % fn_tag
  				pt_csv = sh.PT_CSV(csv_fn, self)
  				pt_csv.open_csv()
  
  				###################################################################
  				# Start with the Municipal maps page
  
  				munic_table = rec_soup.find('table')
  
  				# Get the metadata url
  				h3_font = rec_soup.find('font', text='Municipal Maps ')
  				h3 = h3_font.parent
  				p = h3.find_next_sibling('p')
  				mdata_a = p.find('a')
  				mdata_url = shared.get_anchor_url(mdata_a, page_url)
  
  				# Set the unique_cols
  				unique_cols = collections.OrderedDict()
  				unique_cols['ignore_cols'] = ['.pdf']
  				rec_list = self.process_table(parent_title, munic_table, page_url, unique_cols)
  
  				desc_str = "Municipal maps produced by Manitoba Infrastructure and Transportation, Highway Planning " \
  						   "and Design Branch, Drafting Section."
  
  				# Add the table of the Designated Drain Watercourses page to the CSV
  				for idx, rec in enumerate(rec_list):
  					msg = "Extracting %s of %s records" % (idx + 1, len(rec_list))
  					shared.print_oneliner(msg)
  				
  					for k, v in rec.items():
  						pt_csv.add(k, v)
  
  					pt_csv.add('Source', source)
  					pt_csv.add('Metadata URL', mdata_url)
  					pt_csv.add('Description', desc_str)
  
  					pt_csv.write_dataset()
  					
  				print
  
  				###################################################################
  				# Get the datasets before amalgamation
  
  				# Get the before amalgamation page's URL
  				amal_a = bsoup.find_tags_containing(rec_soup, 'municipal amalgamation', 'a')
  				amal_url = shared.get_anchor_url(amal_a, page_url)
  
  				# Get the before amalgamation page's soup
  				amal_soup = bsoup.get_soup(amal_url)
  
  				# Get the table of datasets
  				amal_table = amal_soup.find('table')
  
  				# Set the unique_cols
  				unique_cols = collections.OrderedDict()
  				unique_cols['ignore_cols'] = ['.gif']
  				rec_list = self.process_table(parent_title, amal_table, amal_url, unique_cols, start_row=6)
  
  				# Add the table of the before amalgamation page to the CSV
  				for idx, rec in enumerate(rec_list):
  					msg = "Extracting %s of %s records" % (idx + 1, len(rec_list))
  					shared.print_oneliner(msg)
  					
  					for k, v in rec.items():
  						pt_csv.add(k, v)
  
  					pt_csv.add('Source', source)
  					pt_csv.add('Metadata URL', mdata_url)
  					pt_csv.add('Description', desc_str)
  
  					pt_csv.write_dataset()
  					
  				print
  
  			elif parent_title == 'Quarter Section Grids':
  				#print 'Quarter Section Grids'
  
  				# Create the CSV file and determining category
  				csv_fn = "MLI_%s_results" % fn_tag
  				pt_csv = sh.PT_CSV(csv_fn, self)
  				pt_csv.open_csv()
  
  				###################################################################
  				# Get the Original Southern Grid
  				south_font = bsoup.find_tags_containing(rec_soup, 'Original Southern Grid', 'font')
  				south_a = south_font.parent
  				south_url = shared.get_anchor_url(south_a, page_url)
  
  				# Remove <br> between Land Use Maps and Municipal Maps
  				south_page = shared.open_webpage(south_url)
  				before_txt = '</CENTER></A>'
  				after_txt = "</A></CENTER>"
  				south_content = south_page.read()
  				#print south_content.find(before_txt)
  				south_content = south_content.replace(before_txt, after_txt)
  
  				# Get the southern grid soup
  				south_soup = BeautifulSoup(south_content, 'html.parser')
  
  				# Get the southern grid table
  				south_table = south_soup.find('table', attrs={'border': '2'})
  
  				# Set the unique_cols
  				unique_cols = collections.OrderedDict()
  				unique_cols['mdata_cols'] = '.doc metadata'
  				unique_cols['ignore_cols'] = ['.gif overview']
  				ignore_rows = ['Go to top']
  				rec_list = self.process_table(parent_title, south_table, south_url, unique_cols, ignore_rows=ignore_rows)
  
  				# Add the table of the southern grid page to the CSV
  				for idx, rec in enumerate(rec_list):
  					msg = "Extracting %s of %s records" % (idx + 1, len(rec_list))
  					shared.print_oneliner(msg)
  					
  					for k, v in rec.items():
  						if k == 'Title':
  							if v == 'All items listed below in one file':
  								v = 'Entire Southern Grid'
  						pt_csv.add(k, v)
  
  					pt_csv.add('Source', source)
  					pt_csv.write_dataset()
  					
  				print
  
  				###################################################################
  				# Get the Manitoba Reference Grid
  				ref_a = bsoup.find_tags_containing(rec_soup, 'Manitoba Reference Grid', 'a')
  				ref_url = shared.get_anchor_url(ref_a, page_url)
  				mdata_url = ref_url.replace("shp", "meta")
  
  				# Get the north grid soup
  				ref_soup = bsoup.get_soup(ref_url)
  				mdata_soup = bsoup.get_soup(mdata_url)
  
  				# Get the metadata link from the map in mdata_soup
  				mdata_area = mdata_soup.find('area')
  				mdata_link = mdata_area['href']
  				mdata_url = urlparse.urljoin(mdata_url, mdata_link)
  
  				# Get the formats from the top table
  				format_table = ref_soup.find('table', attrs={'border': '2'})
  				columns = format_table.find_all('td')
  				formats = [f.text.strip().replace('.', '') for f in columns[2:]]
  
  				# Get the areas on the page and soup up there pages
  				ref_areas = ref_soup.find_all('area')
  				for idx, area in enumerate(ref_areas):
  				
  					msg = "Extracting %s of %s areas" % (idx + 1, len(ref_areas))
  					shared.print_oneliner(msg)
  				
  					# Get the area link
  					area_link = area['href']
  					if area_link == '': continue
  					area_url = urlparse.urljoin(ref_url, area_link)
  
  					#print "area_link: '%s'" % area_link
  
  					# Get the tile name from the shapefile filename
  					base_name = os.path.basename(area_link)
  					base_split = base_name.split("_")
  					tile_name = base_split[2]
  
  					# Get the metadata info
  					mdata_info = self.get_metadata(mdata_url)
  					if mdata_info is not None:
  						for k, v in mdata_info.items():
  							pt_csv.add(k, v)
  
  					title_str = "%s - Reference Grid: %s" % (parent_title, tile_name)
  
  					formats_str = "|".join(formats)
  
  					download_info = shared.get_download_text(formats, area_url)
  					download_str, access_str = download_info.split('|')
  
  					# print "Title: "
  					# print title_str
  					# print "Formats: " + str(formats_str)
  					# print "Metadata URL: " + str(mdata_url)
  					# print "Download: " + str(download_str)
  					# print "Access: " + str(access_str)
  
  					# Add all geospatial data to the pt_csv
  					pt_csv.add('Source', source)
  					pt_csv.add('Title', title_str)
  					pt_csv.add('Access', access_str)
  					pt_csv.add('Type', 'Vector Data')
  					pt_csv.add('Download', download_str)
  					pt_csv.add('Web Page URL', ref_url)
  					pt_csv.add('Metadata URL', mdata_url)
  					pt_csv.add('Spatial Reference', sp_str)
  					pt_csv.add('Available Formats', formats_str)
  
  					pt_csv.write_dataset()
  
  				print
  				###################################################################
  				# Get the Northern Grid Master tiles
  				north_a = bsoup.find_tags_containing(rec_soup, 'Master tiles', 'a')
  				north_url = shared.get_anchor_url(north_a, page_url)
  
  				#print "north_url: " + str(north_url)
  
  				# Get the north grid soup
  				north_soup = bsoup.get_soup(north_url.replace("shp", "meta"))
  
  				# Get the formats from the top table
  				format_table = north_soup.find('table', attrs={'border': '2'})
  				columns = format_table.find_all('td')
  				formats  = [f.text.strip().replace('.', '') for f in columns[2:]]
  
  				# Get the areas on the page and soup up there pages
  				north_areas = north_soup.find_all('area')
  				for idx, area in enumerate(north_areas):
  					msg = "Extracting %s of %s areas" % (idx + 1, len(north_areas))
  					shared.print_oneliner(msg)
  				
  					# Get the tile name
  					if area.has_attr('alt'):
  						id = area['alt']
  						#print "id: " + str(id)
  						# Get the metadata URL
  						mdata_link = area['href']
  						mdata_url = urlparse.urljoin(north_url, mdata_link)
  
  						# Get the metadata info
  						mdata_info = self.get_metadata(mdata_url)
  						if mdata_info is not None:
  							for k, v in mdata_info.items():
  								pt_csv.add(k, v)
  
  						title_str = "%s - Northern Quarter Sections: %s" % (parent_title, id)
  
  						formats_str = "|".join(formats)
  
  						download_info = shared.get_download_text(formats)
  						download_str, access_str = download_info.split('|')
  
  						# Hard-code the description from the metadata
  						desc_str = '''The above grid consists of a series of topologically structured Mastertiles and Subtiles.
  
  						A Mastertile is 4 townships deep (24 miles) and extends province wide from the Manitoba/Saskatchewan boundary to the Manitoba/Ontario boundary. There are
  						presently 25 Mastertiles in the grid with Mastertile 1 being the most northerly full 4 townships in the province and the Mastertile numbers increasing by one for each 4 townships one proceeds southwards.
  
  						A Subtile is a subset of the Mastertile and generally consists of a block 4 townships deep by 10 ranges wide (40 townships in all). The actual number of townships in a Subtile will vary when a particular tile closes on a meridian, an interprovincial boundary, or large natural water body such as Hudson Bay. Each Subtile bears the Mastertile name followed by a letter designation. The letter "A"
  						has been assigned to the most westerly Subtile with Subtiles "B", "C", "D" and so on occurring as one proceeds easterly.'''
  						desc_str = desc_str
  
  						# print "Title: "
  						# print title_str
  						# print "Formats: " + str(formats_str)
  						# print "Metadata URL: " + str(mdata_url)
  						# print "Download: " + str(download_str)
  						# print "Access: " + str(access_str)
  
  						# Add all geospatial data to the pt_csv
  						pt_csv.add('Source', source)
  						pt_csv.add('Title', title_str)
  						pt_csv.add('Description', desc_str)
  						pt_csv.add('Access', access_str)
  						pt_csv.add('Type', 'Vector Data')
  						pt_csv.add('Download', download_str)
  						pt_csv.add('Web Page URL', north_url)
  						pt_csv.add('Metadata URL', mdata_url)
  						pt_csv.add('Spatial Reference', sp_str)
  						pt_csv.add('Available Formats', formats_str)
  
  						pt_csv.write_dataset()
  
  				print
  				
  				###################################################################
  				# Get the Northern Grid Sub-tiles
  				subtiles_a = bsoup.find_tags_containing(rec_soup, 'Sub-tiles', 'a')
  				subtiles_url = shared.get_anchor_url(subtiles_a, page_url)
  
  				subtiles_soup = bsoup.get_soup(subtiles_url)
  
  				# Now get all the pages on the map to access the list of tiles
  				subtiles_areas = subtiles_soup.find_all('area')
  				for area in subtiles_areas:
  					area_link = area['href']
  					area_url = urlparse.urljoin(subtiles_url, area_link)
  
  					# Get the formats from the top table
  					format_table = north_soup.find('table', attrs={'border': '2'})
  					columns = format_table.find_all('td')
  					formats = [f.text.strip().replace('.', '') for f in columns[2:]]
  
  					# Go through each area (tile) on subtiles page
  					area_soup = bsoup.get_soup(area_url)
  					subareas = area_soup.find_all('area')
  
  					for idx, sub in enumerate(subareas):
  						msg = "Extracting %s of %s areas" % (idx + 1, len(subareas))
  						shared.print_oneliner(msg)
  					
  						sub_link = sub['href']
  						sub_url = urlparse.urljoin(area_url, sub_link)
  
  						# Get the tile name from the Zip file
  						#tile_num = re.findall('[0-9][0-9][a-z]', sub_link)
  						title_str = "%s - %s" % (parent_title, os.path.basename(sub_link).replace('_shp.zip', ''))
  
  						# Hard-code the description from the metadata
  						desc_str = '''The above grid consists of a series of topologically structured Mastertiles and Subtiles.
  
  						A Mastertile is 4 townships deep (24 miles) and extends province wide from the Manitoba/Saskatchewan boundary to the Manitoba/Ontario boundary. There are
  						presently 25 Mastertiles in the grid with Mastertile 1 being the most northerly full 4 townships in the province and the Mastertile numbers increasing by one for each 4 townships one proceeds southwards.
  
  						A Subtile is a subset of the Mastertile and generally consists of a block 4 townships deep by 10 ranges wide (40 townships in all). The actual number of townships in a Subtile will vary when a particular tile closes on a meridian, an interprovincial boundary, or large natural water body such as Hudson Bay. Each Subtile bears the Mastertile name followed by a letter designation. The letter "A"
  						has been assigned to the most westerly Subtile with Subtiles "B", "C", "D" and so on occurring as one proceeds easterly.'''
  
  						# Get the metadata URL
  						sub_mdata_url = area_url.replace('_shp', '_meta')
  						sub_mdata_soup = bsoup.get_soup(sub_mdata_url)
  						sub_mdata_area = sub_mdata_soup.find('area')
  						mdata_link = sub_mdata_area['href']
  						mdata_url = urlparse.urljoin(sub_mdata_url, mdata_link)
  
  						formats_str = "|".join(formats)
  
  						download_info = shared.get_download_text(formats)
  						download_str, access_str = download_info.split('|')
  
  						# print "Title: "
  						# print title_str
  						# print "Formats: " + str(formats_str)
  						# print "Metadata URL: " + str(mdata_url)
  						# print "Download: " + str(download_str)
  						# print "Access: " + str(access_str)
  
  						# Add all geospatial data to the pt_csv
  						pt_csv.add('Source', source)
  						pt_csv.add('Title', title_str)
  						pt_csv.add('Description', desc_str)
  						pt_csv.add('Access', access_str)
  						pt_csv.add('Type', 'Vector Data')
  						pt_csv.add('Download', download_str)
  						pt_csv.add('Web Page URL', north_url)
  						pt_csv.add('Metadata URL', mdata_url)
  						pt_csv.add('Spatial Reference', sp_str)
  						pt_csv.add('Available Formats', formats_str)
  
  						pt_csv.write_dataset()
  
  					print
  				###################################################################
  				# Get the Northern Grid Quarter Sections
  				sect_a = bsoup.find_tags_containing(rec_soup, 'Quarter Sections', 'a')
  				sect_url = shared.get_anchor_url(sect_a, page_url)
  				access_str = 'Download/Web Accessible'
  
  				title_str = "%s - %s" % (parent_title, shared.clean_text(sect_a.text))
  
  				# Add all geospatial data to the pt_csv
  				pt_csv.add('Source', source)
  				pt_csv.add('Title', title_str)
  				pt_csv.add('Access', access_str)
  				pt_csv.add('Type', 'Vector Data')
  				pt_csv.add('Download', sect_url)
  				pt_csv.add('Web Page URL', north_url)
  				pt_csv.add('Spatial Reference', sp_str)
  				pt_csv.add('Available Formats', 'SHP')
  
  				pt_csv.write_dataset()
  
  				###################################################################
  				# Get the DLS corner points
  				pnts_a = bsoup.find_tags_containing(rec_soup, 'corner points', 'a')
  				pnts_url = shared.get_anchor_url(pnts_a, page_url)
  				access_str = 'Download/Web Accessible'
  
  				title_str = "%s - %s" % (parent_title, shared.clean_text(pnts_a.text))
  
  				# Add all geospatial data to the pt_csv
  				pt_csv.add('Source', source)
  				pt_csv.add('Title', title_str)
  				pt_csv.add('Access', access_str)
  				pt_csv.add('Type', 'Vector Data')
  				pt_csv.add('Download', pnts_url)
  				pt_csv.add('Web Page URL', north_url)
  				pt_csv.add('Spatial Reference', sp_str)
  				pt_csv.add('Available Formats', 'SHP')
  
  				pt_csv.write_dataset()
  
  			elif parent_title == 'Soil Classification':
  				#print 'Soil Classification'
  
  				# Create the CSV file and determining category
  				csv_fn = "MLI_%s_results" % fn_tag
  				pt_csv = sh.PT_CSV(csv_fn, self)
  				pt_csv.open_csv()
  
  				table = rec_soup.find('table', attrs={'border': '2'})
  				a_list = table.find_all('a')
  
  				for a in a_list:
  				
  					soil_url = shared.get_anchor_url(a, page_url)
  					soil_soup = bsoup.get_soup(soil_url, True)
  
  					a_text = shared.clean_text(a.text)
  
  					if a_text.find('SoilAID') > -1:
  						table = soil_soup.find('table', attrs={'border': '2'})
  
  						table_rows = shared.table_to_dict(table, 0, start_row=6)
  
  						#print table_rows
  
  						for idx, row in enumerate(table_rows):
  							msg = "Extracting %s of %s records" % (idx + 1, len(table_rows))
  							shared.print_oneliner(msg)
  						
  							# Get the title and skip if 'Go to top'
  							title_tag = row['block area']
  							title_str = "%s - %s" % (parent_title, bsoup.get_text(title_tag))
  							if title_str.find('Go to top') > -1: continue
  
  							# Get the metadata info
  							mdata_a = row['.doc'].a
  							mdata_url = shared.get_anchor_url(mdata_a, soil_url)
  							mdata_info = self.get_metadata(mdata_url)
  
  							if isinstance(mdata_info, str):
  								pt_csv.add('Notes', mdata_info)
  							else:
  								# Add the metadata info to the CSV file
  								if mdata_info is not None:
  									for k, v in mdata_info.items():
  										pt_csv.add(k, v)
  
  							# Set the formats
  							formats = ['DXF', 'SHP', 'KMZ']
  
  							# Set the download text and access text
  							download_info = shared.get_download_text(formats)
  							download_str, access_str = download_info.split('|')
  
  							pt_csv.add('Source', source)
  							pt_csv.add('Title', title_str)
  							pt_csv.add('Available Formats', '|'.join(formats))
  							pt_csv.add('Type', 'Vector Data')
  							pt_csv.add('Access', access_str)
  							pt_csv.add('Metadata URL', mdata_url)
  							pt_csv.add('Spatial Reference', sp_str)
  
  							pt_csv.write_dataset()
  
  							#######################################################################
  							# Now get the drain dataset
  							drain_kmz = row['.kmz drains']
  							# Get the title
  							drain_title = "%s - Drains" % title_str
  
  							# Get the download
  							drain_a = drain_kmz.a
  							download_url = shared.get_anchor_url(drain_a, soil_url)
  
  							# Set the formats
  							drain_formats = ['DXF', 'SHP', 'KMZ']
  
  							# Set the download text and access text
  							download_info = shared.get_download_text(drain_formats, download_url)
  							download_str, access_str = download_info.split('|')
  
  							if isinstance(mdata_info, str):
  								pt_csv.add('Notes', mdata_info)
  							else:
  								# Add the metadata info to the CSV file
  								if mdata_info is not None:
  									for k, v in mdata_info.items():
  										pt_csv.add(k, v)
  
  							pt_csv.add('Source', source)
  							pt_csv.add('Title', drain_title)
  							pt_csv.add('Available Formats', 'KMZ')
  							pt_csv.add('Type', 'Vector Data')
  							pt_csv.add('Access', access_str)
  							pt_csv.add('Metadata URL', mdata_url)
  							pt_csv.add('Spatial Reference', sp_str)
  
  							pt_csv.write_dataset()
  
  						print
  					elif a.text.find('SoilSMUF') > -1:
  						# Extract the Soil Map Unit File (SoilSMUF) by Municipality page
  						table = soil_soup.find('table', attrs={'border': '2'})
  
  						unique_cols = collections.OrderedDict()
  						unique_cols['mdata_col'] = '.doc'
  						unique_cols['ignore_cols'] = ['.gif']
  						ignore_rows = ['SMUF Description', 'Go to top']
  						rec_list = self.process_table(parent_title, table, soil_url, unique_cols, ignore_rows=ignore_rows)
  
  						# Add the table on the Forest Inventory page to the CSV
  						for idx, rec in enumerate(rec_list):
  							msg = "Extracting %s of %s records" % (idx + 1, len(rec_list))
  							shared.print_oneliner(msg)
  						
  							for k, v in rec.items():
  								pt_csv.add(k, v)
  
  							pt_csv.add('Source', source)
  							pt_csv.write_dataset()
  							
  						print
  					elif a.text.find('Detailed Soil Map') > -1:
  						# Extract the Detailed Soils (new - 2014) by Municipality page
  						table = soil_soup.find('table', attrs={'border': '2'})
  
  						unique_cols = collections.OrderedDict()
  						unique_cols['mdata_col'] = '.doc'
  						unique_cols['ignore_cols'] = ['.gif']
  						ignore_rows = ['Detailed Soils', 'Go to top']
  						rec_list = self.process_table(parent_title, table, soil_url, unique_cols, ignore_rows=ignore_rows)
  
  						# Add the table on the Forest Inventory page to the CSV
  						for idx, rec in enumerate(rec_list):
  							msg = "Extracting %s of %s records" % (idx + 1, len(rec_list))
  							shared.print_oneliner(msg)
  							
  							for k, v in rec.items():
  								pt_csv.add(k, v)
  
  							pt_csv.add('Source', source)
  							pt_csv.write_dataset()
  							
  						print
  
  			elif parent_title == 'Spatial Referencing':
  				#print 'Spatial Referencing'
  
  				# Create the CSV file and determining category
  				csv_fn = "MLI_%s_results" % fn_tag
  				pt_csv = sh.PT_CSV(csv_fn, self)
  				pt_csv.open_csv()
  
  				table = rec_soup.find('table', attrs={'id': 'table2'})
  
  				unique_cols = collections.OrderedDict()
  				unique_cols['mdata_col'] = '.doc'
  				unique_cols['desc_col'] = 'description'
  
  				table_rows = shared.table_to_dict(table, 0, start_row=2)
  
  				for idx, row in enumerate(table_rows):
  					msg = "Extracting %s of %s records" % (idx + 1, len(table_rows))
  					shared.print_oneliner(msg)
  				
  					# Get the title of the dataset
  					title_str = "%s - %s" % (parent_title, shared.clean_text(row['product'].text))
  
  					if title_str.find('Diagrams') > -1:
  						# The Township Diagrams are tiles
  						formats_str = 'TIF'
  						download_str = 'Multiple Downloads'
  						access_str = 'Download/Web Accessible'
  						desc_str = shared.clean_text(row['description'].text)
  					else:
  
  						# Get the metadata info
  						mdata_a = row['.doc'].a
  						mdata_url = shared.get_anchor_url(mdata_a, page_url)
  						if not mdata_url == '':
  							mdata_info = self.get_metadata(mdata_url)
  							for k, v in mdata_info.items():
  								pt_csv.add(k, v)
  
  						# Get the description
  						desc_str = shared.clean_text(row['description'].text)
  
  						# Get the formats
  						a_list = row['format'].find_all('a')
  						formats = []
  						downloads = []
  						for a in a_list:
  							link = a['href']
  							url = shared.get_anchor_url(a, page_url)
  							if link.find('html') == -1:
  								formats.append(a.text.replace('.', ''))
  								downloads.append(url)
  
  						formats_str = '|'.join([f for f in formats if not f == ''])
  
  						download_info = shared.get_download_text(formats)
  						download_str, access_str = download_info.split('|')
  
  					# print "Title: "
  					# print title_str
  					# print "Formats: " + str(formats_str)
  					# print "Metadata URL: " + str(mdata_url)
  					# print "Download: " + str(download_str)
  					# print "Access: " + str(access_str)
  
  					# Add all geospatial data to the pt_csv
  					pt_csv.add('Source', source)
  					pt_csv.add('Title', title_str)
  					pt_csv.add('Description', desc_str)
  					pt_csv.add('Access', access_str)
  					pt_csv.add('Download', download_str)
  					pt_csv.add('Web Page URL', page_url)
  					pt_csv.add('Metadata URL', mdata_url)
  					pt_csv.add('Spatial Reference', sp_str)
  					pt_csv.add('Available Formats', formats_str)
  
  					pt_csv.write_dataset()
  
  				print
  			elif parent_title == 'Topographic Maps':
  				# Create the CSV file and determining category
  				csv_fn = "MLI_%s_results" % fn_tag
  				pt_csv = sh.PT_CSV(csv_fn, self)
  				pt_csv.open_csv()
  
  				table = rec_soup.find('table', attrs={'border': '2'})
  
  				rows = table.find_all('tr')
  
  				for row in rows:
  				
  					# Get the anchor in this row
  					a = row.a
  					if a is None: continue
  
  					a_text = shared.clean_text(a.text)
  
  					# Get a list of columns from the row
  					cols = row.find_all('td')
  
  					# Get the description from the second column of the row
  					desc_str = shared.clean_text(cols[1].text)
  
  					# If the link is not to a page (.html), then skip it
  					sub_url = shared.get_anchor_url(a, page_url)
  					if sub_url.find('.html') == -1: continue
  
  					sub_soup = bsoup.get_soup(sub_url)
  
  					if a_text == '1:20,000':
  						# This page contains a map with links to more subpages
  						#   with tables with datasets
  						sub_map = sub_soup.find('map')
  						map_info = self.process_map(sub_map, sub_url)
  
  						for area in map_info:
  						
  							area_url = area['Download']
  							area_soup = bsoup.get_soup(area_url)
  							area_table = area_soup.find('table', attrs={'border': '2'})
  							if area_table is None:
  								area_table = area_soup.find('table', attrs={'border': '1'})
  
  							# Process the table
  							unique_cols = collections.OrderedDict()
  							unique_cols['mdata_col'] = '.doc metadata'
  							unique_cols['ignore_cols'] = ['.gif overview']
  							table_rows = self.process_table(parent_title, area_table, area_url, unique_cols, title_prefix='Topo Map')
  
  							# Add the table on the page to the CSV
  							for idx, row in enumerate(table_rows):
  								msg = "Extracting %s of %s records" % (idx + 1, len(table_rows))
  								shared.print_oneliner(msg)
  							
  								# Add the description to the pt_csv
  								pt_csv.add('Description', desc_str)
  
  								for k, v in row.items():
  									pt_csv.add(k, v)
  
  								# Add the description to the pt_csv
  								pt_csv.add('Source', source)
  								pt_csv.add('Description', desc_str)
  
  								pt_csv.write_dataset()
  								
  							print
  
  					elif a_text == '1:20,000 Special':
  						# Get the table with a border
  						sub_table = sub_soup.find('table', attrs={'border': '2'})
  						if sub_table is None:
  							sub_table = sub_soup.find('table', attrs={'border': '1'})
  
  						# Process the table
  						unique_cols = collections.OrderedDict()
  						unique_cols['mdata_col'] = '.doc metadata'
  						unique_cols['ignore_cols'] = ['.gif overview']
  						table_rows = self.process_table(parent_title, sub_table, sub_url, unique_cols, title_prefix='Topo Map')
  
  						# Add the table on the page to the CSV
  						for idx, row in enumerate(table_rows):
  							msg = "Extracting %s of %s records" % (idx + 1, len(table_rows))
  							shared.print_oneliner(msg)
  								
  							# Add the description to the pt_csv
  							pt_csv.add('Description', desc_str)
  
  							for k, v in row.items():
  								pt_csv.add(k, v)
  
  							pt_csv.add('Source', source)
  							pt_csv.write_dataset()
  							
  						print
  
  					elif a_text == '1:20,000 FMU':
  						# Get the table with different formats
  						formats_table = sub_soup.find('table', attrs={'border': '2'})
  						cols = formats_table.find_all('td')
  
  						# Get the formats from the columns in the table
  						formats = []
  						for col in cols[2:3]:
  							col_text = bsoup.get_text(col)
  							formats.append(col_text.upper())
  
  						# Now get the areas in the map
  						sub_map = sub_soup.find('map')
  						areas = sub_map.find_all('area')
  						for idx, area in enumerate(areas):
  							msg = "Extracting %s of %s areas" % (idx + 1, len(areas))
  							shared.print_oneliner(msg)
  						
  							if area.has_attr('href'):
  								area_link = area['href']
  								area_url = urlparse.urljoin(sub_url, area_link)
  
  								if area_url.find('.html') == -1:
  									bsname = os.path.basename(area_url)
  									title = bsname.split('_')[0]
  									title_str = '%s - Topo Map FMU: %s' % (parent_title, title)
  
  									# Add all geospatial data to the pt_csv
  									pt_csv.add('Source', source)
  									pt_csv.add('Title', title_str)
  									pt_csv.add('Access', 'Download/Web Accessible')
  									pt_csv.add('Type', 'Vector Data')
  									pt_csv.add('Download', 'Multiple Downloads')
  									pt_csv.add('Web Page URL', sub_url)
  									pt_csv.add('Available Formats', '|'.join(formats))
  
  									pt_csv.write_dataset()
  									
  						print
  
  					elif a_text == '1:20,000 Seamless':
  						# Get the table on the page
  						table = sub_soup.find('table', attrs={'border': '2'})
  
  						# Process the table
  						unique_cols = collections.OrderedDict()
  						unique_cols['mdata_col'] = '.doc'
  						unique_cols['ignore_cols'] = ['type', '.gif']
  						table_rows = self.process_table(parent_title, table, sub_url, unique_cols)
  
  						# Add the table on the page to the CSV
  						for idx, row in enumerate(table_rows):
  							msg = "Extracting %s of %s records" % (idx + 1, len(table_rows))
  							shared.print_oneliner(msg)
  						
  							# Add the description to the pt_csv
  							pt_csv.add('Description', desc_str)
  
  							for k, v in row.items():
  								pt_csv.add(k, v)
  
  							pt_csv.add('Source', source)
  							pt_csv.write_dataset()
  							
  						print
  
  					if a['href'].find('t50k/index.html') > -1:
  						# This page contains a map with links to more subpages
  						#   with tables with datasets
  						sub_map = sub_soup.find('map')
  						map_info = self.process_map(sub_map, sub_url)
  
  						for area in map_info:
  							area_url = area['Download']
  							area_soup = bsoup.get_soup(area_url)
  							if isinstance(area_soup, int):
  								print "Error Code: %s" % area_soup
  								if area_soup == 404:
  									print "The webpage '%s' cannot be found." % area_url
  									#answer = raw_input("Press enter...")
  									continue
  									
  							# if area_soup is None:
  								# print "The webpage '%s' cannot be found." % area_url
  								# continue
  								
  							if not self.check_result(area_soup, area_url): continue
  							
  							area_table = area_soup.find('table', attrs={'border': '2'})
  							if area_table is None:
  								area_table = area_soup.find('table', attrs={'border': '1'})
  
  							# Process the table
  							unique_cols = collections.OrderedDict()
  							unique_cols['mdata_col'] = '.doc metadata'
  							unique_cols['ignore_cols'] = ['.gif overview']
  							table_rows = self.process_table(parent_title, area_table, area_url, unique_cols, title_prefix='Topo Map')
  
  							# Add the table on the page to the CSV
  							for idx, row in enumerate(table_rows):
  								msg = "Extracting %s of %s records" % (idx + 1, len(table_rows))
  								shared.print_oneliner(msg)
  								
  								# Add the description to the pt_csv
  								pt_csv.add('Description', desc_str)
  
  								for k, v in row.items():
  									pt_csv.add(k, v)
  
  								# Add the description to the pt_csv
  								pt_csv.add('Source', source)
  								pt_csv.add('Description', desc_str)
  
  								pt_csv.write_dataset()
  								
  							print
  
  			elif parent_title == 'Town & Village Plans':
  				#print 'Town & Village Plans'
  
  				print 'No geospatial data, all PDFs or GIFs'
  				
  		# Print ending time
  		end_time = datetime.datetime.now()
  		print "\nExtraction complete at %s." % end_time
  		tot_time = end_time - start_time
  		print "It took %s to complete." % tot_time
  				
  	def extract_municipal(self): #, word=None, category=None, subpage='all'):
  		''' Extracts all municipal pages for Manitoba (specifically Brandon and Winnipeg)
  		:param word: The word used to filter the results.
  		:param category: The category to filter the results.
  		:param subpage: The subpage to run (only for debugging).
  		:return: None
  		'''
  		
  		# Get the parameters
  		word = self.get_arg_val('word')
  		category = self.get_arg('category')
  		subpage = self.get_arg('subpage').get_urltags()[0]
  		
  		#print "subpage: %s" % subpage
  		
  		self.print_title("Extracting Manitoba's municipal pages")
  		
  		start_time = datetime.datetime.now()
  		print "Process started at: %s" % start_time
  		
  		# NOTE: subpage is for debugging only.
  		if subpage is None: subpage = 'all'
  		
  		# Create CSV file
  		csv_fn = "Municipal_results"
  		pt_csv = sh.PT_CSV(csv_fn, self)
  		pt_csv.open_csv()
  		
  		#############################################################################
  		# Extract from Brandon OpenData
  		
  		if subpage == 'bdn_opendata' or subpage == 'all':
  
  			opendata_url = self.pg_grp.get_url('brandon_opendata_url')
  
  			# Get soup
  			opendata_soup = bsoup.get_soup(opendata_url)
  			
  			if self.check_result(opendata_soup, opendata_url, 'Brandon Open Data'):
  
  				table = opendata_soup.find('table', attrs={'id': 'datasetTable'})
  				rows = table.find_all('tr')
  
  				for idx, row in enumerate(rows):
  					anchor = row.find('a')
  					
  					msg = "Extracting %s of %s datasets from Brandon's Open Data page" % (idx + 1, len(rows))
  					shared.print_oneliner(msg)
  
  					if anchor is not None:
  						# Get the parent row
  						tr = anchor.parent.parent
  						tds = tr.find_all('td')
  
  						# Get the title from the first column and the format from the second column
  						title_str = bsoup.get_text(tds[0])
  						ds_type = bsoup.get_text(tds[1])
  
  						# Determine if the dataset is a shapefile
  						if ds_type == 'Shapefile':
  							# Get the download URL and format
  							download_url = shared.get_anchor_url(anchor, opendata_url)
  							formats = ['SHP']
  
  							# Get the access and download text
  							download_info = shared.get_download_text(formats, download_url)
  							download_str, access_str = download_info.split('|')
  
  							pt_csv.add('Source', 'Manitoba Municipal - Brandon Open Data')
  							pt_csv.add('Title', title_str)
  							pt_csv.add('Type', ds_type)
  							pt_csv.add('Web Page URL', opendata_url)
  							pt_csv.add('Download', download_str)
  							pt_csv.add('Access', access_str)
  							pt_csv.add('Available Formats', formats[0])
  
  							pt_csv.write_dataset()
  
  				print
  		##############################################################################
  		# Extract from Brandon ESRI REST Service
  		
  		if subpage == 'bdn_rest' or subpage == 'all':
  
  			rest_url = self.pg_grp.get_url('brandon_rest_url')
  			my_rest = services.PT_REST(rest_url)
  
  			# Get the service data and add it to the CSV file
  			serv_data = my_rest.get_layers()
  			for rec in serv_data:
  				for k, v in rec.items():
  					pt_csv.add(k, v)
  
  				pt_csv.add('Source', 'Manitoba Municipal - Brandon Map Services')
  					
  				pt_csv.write_dataset()
  
  		##############################################################################
  		# Extract from what's left of Brandon GIS
  		
  		if subpage == 'bdn_gis' or subpage == 'all':
  		
  			bgis_url = self.pg_grp.get_url('brandon_gis_url')
  
  			# Get the soup
  			bgis_soup = bsoup.get_soup(bgis_url)
  
  			# Locate the <section> tag with class 'mainContent'
  			section = bgis_soup.find('section', attrs={'class': 'mainContent'})
  
  			# Collect all the <li> tags in the section
  			li_list = section.find_all('li')
  
  			for idx, li in enumerate(li_list):
  				msg = "Extracting %s of %s datasets from Brandon's GIS page" % (idx + 1, len(li_list))
  				shared.print_oneliner(msg)
  			
  				# Get the map page URL
  				map_a = li.find('a')
  				map_url = shared.get_anchor_url(map_a, bgis_url)
  
  				# Ignore the Google Maps since it's only a link to Google's Transit Map and
  				#   not to a City of Brandon dataset
  				if map_url.find('google.ca') > -1: continue
  				elif map_url.find('snowmap') > -1:
  					map_url = 'https://gisapp.brandon.ca/webmaps/snowclearing/index.html'
  
  				# Get the map soup
  				map_soup = bsoup.get_soup(map_url)
  
  				if map_url.find('arcgis') > -1:
  					# If the map is an ArcGIS Online Map
  					data_url = shared.get_arcgis_url(map_url)
  
  					map_json = shared.get_json(data_url)
  
  					title_str = map_json['title']
  					date_str = shared.translate_date(map_json['modified'])
  					desc_str = map_json['description']
  					lic_str = map_json['licenseInfo']
  					sp_str = shared.get_spatialref(map_json)
  					map_type = 'ArcGIS Online Web Map'
  
  				else:
  					# Look for text in the soup to determine the type of map
  					map_type = 'Web Map Application'
  					if str(map_soup).find('x-shockwave-flash') > -1:
  						# The map is a Adobe Flash Web Map
  						map_type = 'Adobe Flash Web Map Application'
  
  
  					# Get the title
  					title = map_soup.find('title')
  					title_str = bsoup.get_text(title)
  
  					# Get the description
  					desc_str = ''
  					desc_tag = map_soup.find('meta', attrs={'name': 'description'})
  					if desc_tag is not None:
  						desc_str = desc_tag['content']
  
  					# Get the date
  					date_str = ''
  					date_div = map_soup.find('div', attrs={'id': 'updateDiv'})
  					if date_div is not None:
  						date_str = bsoup.get_text(date_div).replace('Last Updated: ', '')
  
  					lic_str = ''
  					sp_str = ''
  					data_url = ''
  
  				pt_csv.add('Source', 'Manitoba Municipal - Brandon GIS')
  				pt_csv.add('Title', title_str)
  				pt_csv.add('Type', map_type)
  				pt_csv.add('Description', desc_str)
  				pt_csv.add('Date', date_str)
  				pt_csv.add('Data URL', data_url)
  				pt_csv.add('Web Page URL', bgis_url)
  				pt_csv.add('Licensing', lic_str)
  				pt_csv.add('Spatial Reference', sp_str)
  				pt_csv.add('Web Map URL', map_url)
  				pt_csv.add('Download', 'No')
  				pt_csv.add('Access', 'Viewable/Contact the Province')
  
  				pt_csv.write_dataset()
  				
  			print
  			
  		##############################################################################
  		# Extract from Winnipeg's Catalogue
  		
  		if subpage == 'wpg_catalogue' or subpage == 'all':
  
  			# Get the Winnipeg Catalogue URL
  			search_url = self.pg_grp.get_url('winnipeg_opendata_url')
  			
  			ds_type = 'maps'
  
  			# Set the parameters for the query URL
  			params = collections.OrderedDict()
  			if word is not None and not word == '':
  				params['q'] = word
  			if category is not None and not category == '':
  				cat_urltag = category.get_urltags()[0]
  				if not cat_urltag == 'all':
  					params['category'] = cat_urltag
  			if ds_type is not None and not ds_type == '':
  				params['limitTo'] = ds_type
  			params['sortBy'] = "alpha"
  
  			# Build the query URL
  			query_url = shared.get_post_query(search_url, params)
  
  			# Get the soup
  			soup = bsoup.get_soup(query_url)
  
  			# Get the number of pages from page buttons at bottom of site
  			res_count_div = soup.find('div', attrs={'class': 'browse2-results-title'})
  			results_text = bsoup.get_text(res_count_div)
  			res_lst = results_text.split(" ")
  
  			num_results = res_lst[0]
  
  			page_count = math.ceil(int(num_results) / 10.0)
  			prev_perc = -1
  			record_count = 0
  			record_total = int(num_results)
  
  			print "Number of pages: " + str(page_count)
  
  			for page in range(0, int(page_count)):
  				# Open each iteration of pages:
  				params['page'] = page + 1
  				page_url = shared.get_post_query(search_url, params)
  				page_soup = bsoup.get_soup(page_url)
  
  				# Get all the datasets on the current page (all datasets are in a 'div' with class 'dataset-item')
  				results = page_soup.find_all('div', attrs={'class': 'browse2-result-content'})
  
  				#print "\nPage URL: %s" % page_url
  				#print "Number of results on page: " + str(len(results))
  
  				if len(results) == 0 and record_count == 0:
  					print "No records exist with the given search parameters."
  					print "URL query sample: %s" % query_url
  					return None
  
  				for dataset in results:
  				
  					record_count += 1
  					msg = "Extracting %s of approximately %s results from '%s'" % (record_count, record_total, query_url)
  					shared.print_oneliner(msg)
  
  					# Get the header of the dataset
  					h2 = dataset.find('h2', attrs={'class': 'browse2-result-name'})
  
  					# Get the link to the page
  					webpage_url = shared.get_link(h2)
  
  					# Get the soup of the dataset
  					attrb = ('class', 'downloadsList')
  					ds_soup = bsoup.get_soup(webpage_url)
  					
  					# Get the metadata from the page
  					mdata_info = bsoup.get_page_metadata(ds_soup)
  					
  					#print "\n"
  					#print webpage_url
  					#for k, v in mdata_info.items():
  					#	print "%s: %s" % (k, v)
  					
  					#if mdata_info['page_name'] == 'Map':
  					
  					# Get the title
  					title_str = mdata_info['page_title']
  					title_str = title_str.replace(' | Open Data | City of Winnipeg', '')
  
  					# Get the description
  					desc_str = mdata_info['description']
  
  					# Get the publisher
  					pub_str = bsoup.get_dl_text(ds_soup, 'Department')
  
  					# Get the date
  					date_str = ''
  					date_span = ds_soup.find('span', attrs={'class': 'aboutUpdateDate'})
  					if date_span is not None: date_str = bsoup.get_text(date_span)
  
  					# Get the licence
  					lic_str = bsoup.get_dl_text(ds_soup, 'Licence')
  
  					# Set the available formats as KML, KMZ, SHP and GeoJSON
  					formats = ['KML', 'KMZ', 'SHP', 'GeoJSON']
  					access_str = 'Download/Web Accessible'
  					download_str = 'Multiple Downloads'
  
  					pt_csv.add('Source', 'Manitoba Municipal - Winnipeg Catalogue')
  					pt_csv.add('Title', title_str)
  					pt_csv.add('Type', "Vector File")
  					pt_csv.add('Description', desc_str)
  					pt_csv.add('Date', date_str)
  					pt_csv.add('Publisher', pub_str)
  					pt_csv.add('Licensing', lic_str)
  					pt_csv.add('Web Page URL', webpage_url)
  					pt_csv.add('Download', download_str)
  					pt_csv.add('Available Formats', '|'.join(formats))
  					pt_csv.add('Access', access_str)
  
  					pt_csv.write_dataset()
  
  				print
  		##############################################################################
  		# Extract from Winnipeg's Property Map/Aerial Photography & ServiceStat
  		
  		if subpage == 'wpg_property' or subpage == 'all':
  
  			# Extract Property Map webpage
  			propertymap_url = self.pg_grp.get_url('winnipeg_property_url')
  
  			# Get the soup
  			property_soup = bsoup.get_soup(propertymap_url, True, "legenditem")
  
  			# Get the layers
  			property_layers = property_soup.find_all('table', attrs={'class': 'legenditem'})
  
  			prop_recs = self.process_googleapi(property_layers, "property")
  
  			for r in prop_recs:
  			
  				for k, v in r.items():
  					pt_csv.add(k, v)
  
  				pt_csv.add('Source', 'Manitoba Municipal - Winnipeg Interactive Maps')
  				pt_csv.add('Web Map URL', propertymap_url)
  				pt_csv.add('Web Page URL', 'http://winnipeg.ca/ppd/maps.stm')
  				pt_csv.add('Download', 'No')
  				pt_csv.add('Access', 'Viewable/Contact the Province')
  
  				pt_csv.write_dataset()
  
  			# Extract the ServiceStat page
  			servstat_url = self.pg_grp.get_url('servstat_url')
  
  			# Get the soup
  			attrb = ('id', 'wpgmap_legend')
  			servstat_soup = bsoup.get_soup(servstat_url, True, attrb)
  			
  			if self.check_result(servstat_soup, servstat_url, 'ServiceStat Page'):
  
  				# Get the legend and layers
  				legend_div = servstat_soup.find('div', attrs={'id': 'wpgmap_legend'})
  				servstat_layers = legend_div.find_all('a')
  
  				stats_recs = self.process_googleapi(servstat_layers, "servstat")
  
  				for r in stats_recs:
  					for k, v in r.items():
  						pt_csv.add(k, v)
  
  					pt_csv.add('Source', 'Manitoba Municipal - Winnipeg Interactive Maps')
  					pt_csv.add('Web Map URL', servstat_url)
  					pt_csv.add('Download', 'No')
  					pt_csv.add('Access', 'Viewable/Contact the Province')
  
  					pt_csv.write_dataset()
  
  		pt_csv.close_csv()
  		
  		# Print ending time
  		end_time = datetime.datetime.now()
  		print "\nExtraction complete at %s." % end_time
  		tot_time = end_time - start_time
  		print "It took %s to complete." % tot_time
  	
  	def extract_services(self):
  		''' Method to extract all Manitoba provincial map services.
  		:return: None
  		'''
  
  		self.print_log("\nExtracting from %s" % self.pg_grp.get_title())
  		
  		self.print_title("Extracting Manitoba's map services")
  		
  		start_time = datetime.datetime.now()
  		print "Process started at: %s" % start_time
  
  		# Create the CSV file
  		csv_fn = "MapServices_results"
  		pt_csv = sh.PT_CSV(csv_fn, self)
  		pt_csv.open_csv()
  
  		##################################################################
  		# Extract the Manitoba's ESRI REST Services
  
  		# Get the petroleum service url
  		petrol_url = self.pg_grp.get_url('petrol_url')
  
  		self.print_log("URL: %s" % petrol_url)
  
  		# Get a list of REST services
  		petrol_rest = services.PT_REST(petrol_url)
  
  		# Get the Petroleum service and add it to the CSV file
  		petrol_data = petrol_rest.get_layers()
  		for rec in petrol_data:
  			for k, v in rec.items():
  				pt_csv.add(k, v)
  
  			pt_csv.add('Source', 'Manitoba Map Services')
  			pt_csv.write_dataset()
  
  		# Get a list of the services2 services
  		rest_url = self.pg_grp.get_url('rest_url')
  		serv_rest = services.PT_REST(rest_url)
  
  		# Get the other service and add it to the CSV file
  		serv_data = serv_rest.get_layers()
  		for rec in serv_data:
  			for k, v in rec.items():
  				pt_csv.add(k, v)
  
  			pt_csv.add('Source', 'Manitoba Map Services')
  			pt_csv.write_dataset()
  
  		pt_csv.close_csv()
  		
  		# Print ending time
  		end_time = datetime.datetime.now()
  		print "\nExtraction complete at %s." % end_time
  		tot_time = end_time - start_time
  		print "It took %s to complete." % tot_time
  
  def main():
  	# parser.add_argument("-t", "--tool", help="The tool to use: %s" % ', '.join(tool_list))
  	# parser.add_argument("-w", "--word", help="The key word(s) to search for.")
  	# parser.add_argument("-f", "--format", help="The format(s) to search for.")
  	# parser.add_argument("-c", "--category", help="The category(ies) to search for.")
  	# parser.add_argument("-d", "--downloadable", help="Determines wheter to get only downloadable datasets.")
  	# parser.add_argument("-l", "--html", help="The HTML file to scrape (only for OpenData website).")
  	# parser.add_argument("-s", "--silent", action='store_true', help="If used, no extra parameters will be queried.")
  
  	ext = Extractor()
  
  	try:
  		pages = ext.get_pagelist()
  
  		parser = argparse.ArgumentParser()
  		parser.add_argument("-p", "--page", help="The page to extract: %s or all" % ', '.join(pages.keys()))
  		parser.add_argument("-c", "--category", help="The category for the MLI sites.")
  		parser.add_argument("-s", "--silent", action='store_true',
  							help="If used, no extra parameters will be queried.")
  		args = parser.parse_args()
  		# print args.echo
  
  		# print "province: " + str(args.province)
  		# print "format: " + str(args.format)
  
  		page = args.page
  		params = collections.OrderedDict()
  		params['cat'] = args.category
  		silent = args.silent
  
  		if page is None:
  			answer = raw_input(
  				"Please enter the page you would like to use (%s or all): " % ', '.join(pages.keys()))
  			if not answer == "":
  				page = answer.lower()
  			else:
  				print "\nERROR: Please specify a web page."
  				print "Exiting process."
  				sys.exit(1)
  
  		page = page.lower()
  
  		#print page
  
  		ext.set_page(page)
  		ext.set_params(params)
  		ext.run()
  
  	except Exception, err:
  		ext.print_log('ERROR: %s\n' % str(err))
  		ext.print_log(traceback.format_exc())
  		ext.close_log()
  
  if __name__ == '__main__':
  	sys.exit(main())