text: #!/usr/bin/env python3
  # coding: utf-8
  #
  '''
  TODO:
  test metadata
  Normalization:
  	cannabinoids total
  	sample type
  	test metadata
  
  use skip_this_file more
  '''
  DATA_ROW_FIELDS = [
  	'Test Result UID',
  	'Sample Name',
  	'Sample Type',
  	'Receipt Time',
  	'Test Time',
  	'Provider',
  	'cis-Nerolidol',
  	'trans-Nerolidol',
  	'trans-Nerolidol 1',
  	'trans-Nerolidol 2',
  	'trans-Ocimene',
  	'3-Carene',
  	'Camphene',
  	'Caryophyllene Oxide',
  	'Eucalyptol',
  	'Geraniol',
  	'Guaiol',
  	'Isopulegol',
  	'Linalool',
  	'Ocimene',
  	'Terpinolene',
  	'alpha-Bisabolol',
  	'alpha-Humulene',
  	'alpha-Pinene',
  	'alpha-Terpinene',
  	'beta-Caryophyllene',
  	'beta-Myrcene',
  	'beta-Ocimene',
  	'beta-Pinene',
  	'delta-Limonene',
  	'gamma-Terpinene',
  	'p-Cymene',
  	'delta-9 THC-A',
  	'delta-9 THC',
  	'delta-8 THC',
  	'THC-A',
  	'THCV',
  	'CBN',
  	'CBD-A',
  	'CBD',
  	'CBDV',
  	'CBDV-A',
  	'delta-9 CBG-A',
  	'delta-9 CBG',
  	'CBC',
  	'Moisture Content',
  ]
  import re, os, csv, argparse, json, urllib, datetime, logging
  from lxml import html
  from lxml import etree
  import dateparser.search as dateparser_search
  
  parser = argparse.ArgumentParser(argument_default=False, description='Clean raw lab data.')
  parser.add_argument('database', nargs='?', default='downloader/database_dump/', help='The location of the database dump.')
  parser.add_argument('--verbose', '-v', action='count', default=0, help='Turn on verbose mode.')
  parser.add_argument('--json', action='store_true', help='Export as JSON.')
  parser.add_argument('--csv', action='store_true', help='Export as CSV.')
  parser.add_argument('--log', help='Logfile path. If omitted, stdout is used.')
  parser.add_argument('--debug', '-d', action='store_true', help='Log all messages including debug.')
  parser.add_argument('--force-terpenes', action='store_true', help='Skip all samples without a terpene profile.')
  parser.add_argument('--force-cannabinoids', action='store_true', help='Skip all samples without a cannabinoid profile.')
  parser.add_argument('--placeholder-csv', default='', help='CSV only: The placeholder to use when no value is present.')
  args = parser.parse_args()
  
  if args.debug:
  	loglevel = logging.DEBUG
  elif args.verbose:
  	loglevel = logging.INFO
  else:
  	loglevel = logging.WARNING
  
  if args.log:
  	logging.basicConfig(filename=args.log, filemode='a', level=loglevel)
  else:
  	logging.basicConfig(level=loglevel)
  
  def get_single_value(tree, xpath, fallback=None, errlevel=logging.CRITICAL, errmsg=None, errparams=[], join_multi=False):
  	raw_value = tree.xpath(xpath)
  	if len(raw_value) == 1:
  		if type(raw_value[0]) == str or type(raw_value[0]) == etree._ElementUnicodeResult:
  			return raw_value[0].strip()
  		else:
  			return raw_value[0]
  	else:
  		if join_multi == False:
  			if errmsg:
  				logging.log(errlevel, errmsg, *errparams)
  			return fallback
  		else:
  			if type(join_multi) == str:
  				return join_multi.join(raw_value).strip()
  			else:
  				return ''.join(raw_value).strip()
  
  def normalize_number(numberstring, base=10, comma=False, separator=False, compress=False):
  	dots = numberstring.count('.')
  	commas = numberstring.count(',')
  	# multiple = more than 1
  	# single = 1
  	# zero = 0
  	## multiple commas, multiple dots:		ERROR
  	if commas > 1 and dots > 1:
  		raise ValueError("could not convert string to float: '{}'".format(numberstring))
  	# AMERICAN
  	## multiple commas, single dot:			remove commas, float
  	## multiple commas, zero dots:			remove commas, float (int)
  	## no commas, single dot:				float
  	if (commas > 1 and dots <= 1) or (commas == 0 and dots == 1):
  		decimal = '.'
  	# EUROPEAN
  	## single comma, multiple dots:			remove dots, float
  	## zero commas, multiple dots:			remove dots, float (int)
  	## single comma, no dots:				float
  	if (dots > 1 and commas <= 1) or (dots == 0 and commas == 1):
  		decimal = ','
  	numberstring = re.sub(r'[^0-9{}]'.format(decimal), '', numberstring)
  	numberstring = numberstring.replace(decimal, '.')
  	result = float(numberstring)
  	if compress:
  		if result == '0.0':
  			return '0'
  		elif result.startswith('0.'):
  			return result[1:]
  	return result
  
  def csv_escape(data):
  	return '"{}"'.format(data)
  
  def write_to_csv(filepath, fieldnames, data):
  	if os.path.exists(filepath):
  		writeheader = False
  	else:
  		writeheader = True
  	with open(filepath, 'a', encoding='utf-8') as writefile:
  		writefile_writer = csv.DictWriter(writefile, fieldnames=fieldnames, restval=args.placeholder_csv, lineterminator='\n')
  		if writeheader:
  			writefile_writer.writeheader()
  		if type(data) != list:
  			data = [data]
  		for data_row in data:
  			writefile_writer.writerow(data_row)
  
  def test_match(xpath_here):
  	matches = tree.xpath(xpath_here)
  	if len(matches) > 0:
  		logging.debug('Match!')
  	for i in matches:
  		logging.debug('Tag: %s', i.tag)
  		logging.debug('Attribs: %s', i.attrib)
  
  logging.debug('Loading configurations . . .')
  
  xpath_provider_page = """"""
  
  xpath_festival_page = """"""
  
  # Finds the list items of the terpene test
  xpath_terpenes_1 =				'/html/body/ui-view/div/md-content/ui-view/div/md-content/div/md-card[@ng-if="Sample.details.terpeneTestComplete"]/div/div/md-table-container/table[thead/tr[th[1][text()="Terpene"]][th[2][text()="Amount"]]]/tbody/tr'
  
  # Finds the amount of total terpenes present in the sample
  xpath_terpenes_total =			'/html/body/ui-view/div/md-content/ui-view/div/md-content/div/md-card[@ng-if="Sample.details.terpeneTestComplete"]/md-card-header/md-card-header-text/span[text()=" Total Terpenes "]/preceding-sibling::span[@class="md-title"]/text()'
  
  # Finds the list items of the cannabinoid test
  xpath_cannabinoids_1 =			'/html/body/ui-view/div/md-content/ui-view/div/md-content/div/md-card[@ng-if="Sample.details.potencyTestComplete"]/div/div/md-table-container/table[thead/tr[th[1][text()="Cannabinoid"]][th[2][text()="Amount"]][th[3][text()="Uncertainty"]]]/tbody/tr'
  
  # Finds the amount of total THC present in the sample
  xpath_thc_total =				'/html/body/ui-view/div/md-content/ui-view/div/md-content/div/md-card[@ng-if="Sample.details.potencyTestComplete"]/md-card-header/md-card-header-text/span[text()="Total THC"][@ng-show="Sample.details.totalTHC"]/preceding-sibling::span[1][@ng-show="Sample.details.totalTHC"]/text()'
  
  # Finds the amount of total CBD present in the sample
  xpath_cbd_total =				'/html/body/ui-view/div/md-content/ui-view/div/md-content/div/md-card[@ng-if="Sample.details.potencyTestComplete"]/md-card-header/md-card-header-text/span[text()="Total CBD"][@ng-show="Sample.details.totalCBD"]/preceding-sibling::span[1][@ng-show="Sample.details.totalCBD"]/text()'
  
  # Finds the type of the sample
  xpath_sample_type =				'/html/body/ui-view/div/md-content/ui-view/div/md-content/div[1]/md-card[1]/md-card-header/md-card-header-text/span[1][following-sibling::span[1][@class="md-title"]]/text()'
  
  # Finds the name of the sample
  xpath_sample_name =				'/html/body/ui-view/div/md-content/ui-view/div/md-content/div[1]/md-card[1]/md-card-header/md-card-header-text/span[@class="md-title"]/text()'
  
  # Finds the provider of the sample
  xpath_sample_provider =			'/html/body/ui-view/div/md-content/ui-view/div/md-content/div[1]/md-card[1]/md-card-header/md-card-header-text/span[@class="md-subhead"][@ng-if="Sample.details.clientInformation.clientId"]/a/text()'
  xpath_sample_provider_anon =	'/html/body/ui-view/div/md-content/ui-view/div/md-content/div[1]/md-card[1]/md-card-header/md-card-header-text/span[@class="md-subhead"][@ng-if="!Sample.details.clientInformation.clientId"]/text()'
  
  xpath_test_uid = """/html/body/ui-view/div/md-content/ui-view/div/md-content/div/md-card/md-card-content/a[
  						starts-with(
  							translate(
  								normalize-space(@href),
  								'ABCDEFGHIJKLMNOPQRSTUVWXYZ',
  								'abcdefghijklmnopqrstuvwxyz'
  							),
  							'/results/samples/edit/'
  						)
  					]/@href"""
  
  # Finds the timestamp of the test of the sample
  xpath_time_tested =				'/html/body/ui-view/div/md-content/ui-view/div/md-content/div[2]/md-card[1]/md-card-content/md-list/md-list-item/span/h3[following-sibling::p[text()="Date Tested"]]/text()'
  # Finds the timestamp of receipt of the sample
  xpath_time_received =			'/html/body/ui-view/div/md-content/ui-view/div/md-content/div[2]/md-card[1]/md-card-content/md-list/md-list-item/span/h3[following-sibling::p[text()="Date Received"]]/text()'
  
  terpenes = {
  	'3-Carene':				re.compile(r'^(delta)?[-_/\s.]*(3|Three|Tri)[-_/\s.]*Carene$',	re.IGNORECASE),
  	'Camphene':				re.compile(r'^Camphene$',										re.IGNORECASE),
  	'Caryophyllene Oxide':	re.compile(r'^Caryophyllene[-_/\s.]*Oxide$',					re.IGNORECASE),
  	'Eucalyptol':			re.compile(r'^Eucalyptol$',										re.IGNORECASE),
  	'Farnesene': 			re.compile(r'^Farnesene$',										re.IGNORECASE),
  	'Geraniol':				re.compile(r'^Geraniol$',										re.IGNORECASE),
  	'Guaiol':				re.compile(r'^Guaiol$',											re.IGNORECASE),
  	'Isopulegol':			re.compile(r'^(\(-\)[-_/\s.]+)?Isopulegol$',					re.IGNORECASE),
  	'Linalool':				re.compile(r'^Linalool$',										re.IGNORECASE),
  	'Ocimene':				re.compile(r'^Ocimene$',										re.IGNORECASE),
  	'Terpinolene':			re.compile(r'^Terpinolene$',									re.IGNORECASE),
  	'alpha-Bisabolol':		re.compile(r'^(alpha|A|α)[-_/\s.]*Bisabolol$',					re.IGNORECASE),
  	'alpha-Humulene':		re.compile(r'^(alpha|A|α)?[-_/\s.]*Humulene$',					re.IGNORECASE),
  	'alpha-Pinene':			re.compile(r'^(alpha|A|α)[-_/\s.]*Pinene$',						re.IGNORECASE),
  	'beta-Pinene':			re.compile(r'^(beta|B|β)[-_/\s.]*Pinene$',						re.IGNORECASE),
  	'alpha-Terpinene':		re.compile(r'^(alpha|A|α)[-_/\s.]*Terpinene$',					re.IGNORECASE),
  	'beta-Caryophyllene':	re.compile(r'^(beta|B|β)?[-_/\s.]*Caryophyllene$',				re.IGNORECASE),
  	'beta-Myrcene':			re.compile(r'^(beta|B|β)?[-_/\s.]*Myrcene$',					re.IGNORECASE),
  	'beta-Ocimene':			re.compile(r'^(beta|B|β)[-_/\s.]*Ocimene$',						re.IGNORECASE),
  	'cis-Nerolidol':		re.compile(r'^(cis)[-_/\s.]*Nerolidol$',						re.IGNORECASE),
  	'delta-Limonene':		re.compile(r'^(delta|D|δ)?[-_/\s.]*Limonene$',					re.IGNORECASE),
  	'gamma-Terpinene':		re.compile(r'^(gamma|G|Y|γ)[-_/\s.]*Terpinene$',				re.IGNORECASE),
  	'p-Cymene':				re.compile(r'^(p)[-_/\s.]*Cymene$',								re.IGNORECASE),
  	'trans-Nerolidol':		re.compile(r'^(trans)[-_/\s.]*Nerolidol$',						re.IGNORECASE),
  	'trans-Nerolidol 1':	re.compile(r'^(trans)[-_/\s.]*Nerolidol[-_/\s.]*1$',			re.IGNORECASE),
  	'trans-Nerolidol 2':	re.compile(r'^(trans)[-_/\s.]*Nerolidol[-_/\s.]*2$',			re.IGNORECASE),
  	'trans-Ocimene':		re.compile(r'^(trans)[-_/\s.]*Ocimene$',						re.IGNORECASE),
  }
  terpenes['Terpene TOTAL'] = re.compile(r'^(Terpene[-_/\s.]*TOTAL|Total[-_/\s.]*[-_/\s.]*Terpenes)',re.IGNORECASE)
  
  cannabinoids = {
  	'delta-9 THC-A':		re.compile(r'^(delta|Δ|∆)[-_/\s.]*9[-_/\s.]*THC[-_/\s.]*A$',	re.IGNORECASE),
  	'delta-9 THC':			re.compile(r'^(delta|Δ|∆)[-_/\s.]*9[-_/\s.]*THC$',				re.IGNORECASE),
  	'CBN':					re.compile(r'^CBN$',											re.IGNORECASE),
  	'CBD-A':				re.compile(r'^CBD[-_/\s.]*A$',									re.IGNORECASE),
  	'CBD':					re.compile(r'^CBD$',											re.IGNORECASE),
  	'CBDV':					re.compile(r'^CBDV$',											re.IGNORECASE),
  	'CBDV-A':				re.compile(r'^CBDV[-_/\s.]*A$',									re.IGNORECASE),
  	'delta-9 CBG-A':		re.compile(r'^((delta|Δ|∆)[-_/\s.]*9[-_/\s.]*)?CBG[-_/\s.]*A$',	re.IGNORECASE),
  	'delta-9 CBG':			re.compile(r'^((delta|Δ|∆)[-_/\s.]*9[-_/\s.]*)?CBG$',				re.IGNORECASE),
  	'CBC':					re.compile(r'^CBC$',											re.IGNORECASE),
  	'THCV':					re.compile(r'^THCV$',											re.IGNORECASE),
  	'delta-8 THC':			re.compile(r'^(delta|Δ|∆)[-_/\s.]*8[-_/\s.]*THC$',				re.IGNORECASE),
  	'Moisture Content':		re.compile(r'^Moisture[-_/\s.]*[-_/\s.]+Content$',				re.IGNORECASE),
  	'THC-A':				re.compile(r'^THC[-_/\s.]*A$',									re.IGNORECASE),
  }
  cannabinoids['THC TOTAL'] = re.compile(r'^THC[-_/\s.]*TOTAL',								re.IGNORECASE)
  cannabinoids['delta-9 THC TOTAL'] = re.compile(r'^(delta|Δ|∆)[-_/\s.]*9[-_/\s.]*THC[-_/\s.]*TOTAL$',re.IGNORECASE)
  cannabinoids['CBD TOTAL'] = re.compile(r'^CBD[-_/\s.]*TOTAL',								re.IGNORECASE)
  cannabinoids['CBG TOTAL'] = re.compile(r'^CBG[-_/\s.]*TOTAL',								re.IGNORECASE)
  cannabinoids['Activated TOTAL'] = re.compile(r'^Activated[-_/\s.]*TOTAL',					re.IGNORECASE)
  cannabinoids['Cannabinoids TOTAL'] = re.compile(r'^TOTAL[-_/\s.]*CANNABINOIDS',				re.IGNORECASE)
  
  sample_types = {
  	'Flower':				re.compile(r'^Flowers?$',										re.IGNORECASE),
  	'Concentrate':			re.compile(r'^Concentrates?$',									re.IGNORECASE),
  	'Edible':				re.compile(r'^Edibles?$',										re.IGNORECASE),
  	'Liquid':				re.compile(r'^Liquids?$',										re.IGNORECASE),
  	'Topical':				re.compile(r'^Topicals?$',										re.IGNORECASE),
  }
  sample_types['Edible Concentrate'] = re.compile(r'^Edible[-_/\s.]*Concentrate$',			re.IGNORECASE)
  sample_types['Infusion'] =				re.compile(r'^Infusion$',							re.IGNORECASE)
  
  re_test_uid = re.compile(r'^/results/samples/edit/(?P<uid>[_a-zA-Z0-9]+)$')
  
  # Parses an american timestamp into its meaningful parts
  re_date =			re.compile(r'^(?P<date>\s*(?P<month>(0?[0-9]|1[0-2]))\s*[-\./:]?\s*(?P<day>(0?[0-9]|1[0-9]|2[0-9]|3[0-1]))\s*[-\./:]?\s*(?P<year>(2\d{3}|[013-9][0-9])))$', re.IGNORECASE)
  
  # Match a percentage value
  re_percentageValue = re.compile(r'^[0-9.,]+\s*%$')
  # Match a percentage value
  re_zeroPercentageValue = re.compile(r'^<\s*[0-9.,]+\s*%$')
  # Match a percentage value at the beginning of the string
  re_percentageValueBeginning = re.compile(r'^[0-9.,]+\s*%')
  # Match a (zero) percentage value at the beginning of the string
  re_zeroPercentageValueBeginning = re.compile(r'^<\s*[0-9.,]+\s*%')
  
  database = {
  }
  
  # database = {
  # 	"name": "Psilabs",
  # 	"samples": []
  # }
  
  providers = []
  sample_types_all = []
  empty_terpenes_counter = 0
  empty_cannabinoid_counter = 0
  
  sample_database_CSVfile = 'results.csv'
  sample_database_JSONfile = 'results.json'
  
  if input('\nDo you want to delete the old result and log files? (y/n) ').lower() == 'y':
  	if os.path.exists(filename+'.csv'):
  		os.remove(filename+'.csv')
  	if os.path.exists(sample_database_CSVfile):
  		os.remove(sample_database_CSVfile)
  	if os.path.exists(sample_database_JSONfile):
  		os.remove(sample_database_JSONfile)
  
  print('Before we start, a heads up:',
  		'I will try to extract any terpene and cannabinoid profiles present as exact as possible. Samples which have values in ppm or mg units are skipped.',
  		'If a specific terpene is not present, I will ignore it for that page',
  		sep='\n'
  		)
  if input('\nDo you want to start? (y/n) ').lower() != 'y':
  	exit('Aborted.')
  
  if args.json:
  	with open(sample_database_JSONfile, "w", encoding="utf-8") as databases_file:
  		databases_file.write('{"name":"Psilabs"')
  		databases_file.write(',"samples":{')
  
  logging.debug('Entering main loop . . .')
  
  type_folders = sorted(os.listdir(os.path.expanduser(args.database)))
  is_first_type = True
  for type_index, type_folder in enumerate(type_folders):
  	is_first_sample = True
  	file_list = sorted(os.listdir(os.path.join(os.path.expanduser(args.database),type_folder)))
  	if args.json:
  		with open(sample_database_JSONfile, "a", encoding="utf-8") as databases_file:
  			if not is_first_type:
  				databases_file.write(',')
  			databases_file.write('"{}":['.format(type_folder))
  			is_first_type = False
  	for file_index, file_name in enumerate(file_list):
  		raw_sample_file_name = os.path.join(type_folder, file_name)
  		logging.debug('#'*80)
  
  		logging.info('%s: Started parsing.', raw_sample_file_name)
  		with open(os.path.join(os.path.expanduser(args.database),raw_sample_file_name),encoding='utf-8') as raw_sample_file:
  			tree = html.fromstring(raw_sample_file.read())
  
  		skip_this_file = False
  
  
  		# 0 Test Data terpenes
  		raw_terpenes_1 = tree.xpath(xpath_terpenes_1)
  		terpenes_data = {}
  		non_percentage_numbers = False
  		# if len(raw_terpenes_1) > 0:
  		# 	logging.warning('%s: Both terpenes queries match!', raw_sample_file_name)
  		if 0 == len(raw_terpenes_1):
  			logging.warning('%s: No terpenes.', raw_sample_file_name)
  		else:
  			for i, raw_terpene in enumerate(raw_terpenes_1, 1):
  
  				# AMOUNT
  				raw_terpene_amount = get_single_value(
  					tree=raw_terpene,
  					xpath='td[2]/text()',
  					fallback=''
  				)
  				terpene_amount_match = re_percentageValue.match(raw_terpene_amount)
  				terpene_zeroamount_match = re_zeroPercentageValue.match(raw_terpene_amount)
  				if terpene_amount_match:
  					terpene_amount_match_object = terpene_amount_match
  					try:
  						terpene_amount = normalize_number(
  							numberstring=raw_terpene_amount[terpene_amount_match.start():terpene_amount_match.end()]
  						)
  					except ValueError as e:
  						logging.warning('%s: Terpenes number error: %s (at list index %d).', raw_sample_file_name, raw_terpene_amount, i)
  						continue
  				elif terpene_zeroamount_match:
  					terpene_amount_match_object = terpene_zeroamount_match
  					terpene_amount = 0.0
  				else:
  					non_percentage_numbers = True
  					logging.info('%s: Non-percentage terpene (at list index %d).', raw_sample_file_name, i)
  					continue
  
  				# NAME
  				## TODO: we could do levenshtein- and typewriterdistance (en-US) here
  				original_terpene_name = get_single_value(
  					tree=raw_terpene,
  					xpath='td[1]/text()',
  					fallback=''
  				)
  				regex_matched = False
  				for terpene_name in terpenes.keys():
  					terpene_regex = terpenes[terpene_name]
  					logging.debug('%s: Trying Regex "%s".', raw_sample_file_name, terpene_regex.pattern)
  					terpene_match = terpene_regex.match(original_terpene_name)
  					if terpene_match:
  						logging.debug('%s: Terpene matched Regex "%s".', original_terpene_name, terpene_regex)
  						if regex_matched:
  							# Match more than one regex?
  							logging.error('%s: Terpene %s matches multiple patterns (at list index %d).', raw_sample_file_name, terpene_name, i)
  							skip_this_file = True
  						else:
  							logging.debug('%s: Regex matched first time: "%s".', raw_sample_file_name, terpene_regex.pattern)
  							regex_matched = True
  						if terpene_name in terpenes_data:
  							# Multiple match same regex?
  							logging.error('%s: Terpene %s already recorded (at list index %d).', raw_sample_file_name, terpene_name, i)
  							skip_this_file = True
  						else:
  							terpenes_data[terpene_name] = terpene_amount
  
  				if original_terpene_name is None:
  					logging.warning('%s: Terpene empty at index %d.', raw_sample_file_name, i)
  				elif not regex_matched:
  					# Match none?
  					logging.error('%s: Terpene did not match anything: %s (list index %d).', raw_sample_file_name, original_terpene_name, i)
  			if terpenes_data == {}:
  				logging.debug('%s: No terpenes were added (%d were found).', raw_sample_file_name, len(raw_terpenes_1))
  		if args.force_terpenes and terpenes_data == {}:
  			skip_this_file = True
  
  		# 1 Test Data Terpenes Total
  		terpene_total = None
  		raw_terpene_total = get_single_value(
  			tree=tree,
  			xpath=xpath_terpenes_total,
  			fallback='',
  			errlevel=logging.ERROR,
  			errmsg='%s: Terpenes TOTAL was not found',
  			errparams=[raw_sample_file_name]
  		)
  		terpene_amount_match = re_percentageValue.match(raw_terpene_total)
  		if terpene_amount_match:
  			try:
  				terpene_total = normalize_number(
  					numberstring=raw_terpene_total
  				)
  			except ValueError as e:
  				logging.warning('%s: Terpene-Total number error: %s.', raw_sample_file_name, raw_terpene_total)
  		else:
  			logging.info('%s: Non-percentage terpene-total: %s.', raw_sample_file_name, raw_terpene_total)
  			continue
  
  		# 1 Test Data Cannabinoids
  		raw_cannabinoids_1 = tree.xpath(xpath_cannabinoids_1)
  		cannabinoid_data = {}
  		# if len(raw_cannabinoids_1) > 0:
  		# 	logging.warning('%s: Both cannabinoid queries match!', raw_sample_file_name)
  		if 0 == len(raw_cannabinoids_1):
  			logging.debug('%s: No potency.', raw_sample_file_name)
  		else:
  			for i, raw_cannabinoid in enumerate(raw_cannabinoids_1, 1):
  
  				# AMOUNT
  				raw_cannabinoid_amount = get_single_value(
  					tree=raw_cannabinoid,
  					xpath='td[2]/text()',
  					fallback=''
  				)
  				cannabinoid_amount_match = re_percentageValue.match(raw_cannabinoid_amount)
  				if cannabinoid_amount_match:
  					try:
  						cannabinoid_amount = normalize_number(
  							numberstring=raw_cannabinoid_amount[cannabinoid_amount_match.start():cannabinoid_amount_match.end()]
  						)
  					except ValueError as e:
  						logging.warning('%s: Cannabinoid number error: %s (at list index %d).', raw_sample_file_name, raw_cannabinoid_info, i)
  						continue
  				else:
  					logging.info('%s: Non-percentage cannabinoid (at list index %d).', raw_sample_file_name, i)
  					continue
  
  				# NAME
  				## TODO: we could do levenshtein- and typewriterdistance (en-US) here
  				original_cannabinoid_name = get_single_value(
  					tree=raw_cannabinoid,
  					xpath='td[1]/text()',
  					fallback=''
  				)
  
  				logging.debug('####################NEW CANNABINOID: %s #####################', original_cannabinoid_name)
  				regex_matched = False
  				for cannabinoid_name in cannabinoids.keys():
  					cannabinoid_regex = cannabinoids[cannabinoid_name]
  					logging.debug('%s: Trying Regex "%s".', raw_sample_file_name, cannabinoid_regex.pattern)
  					cannabinoid_match = cannabinoid_regex.match(original_cannabinoid_name)
  					if cannabinoid_match:
  						logging.debug('%s: Cannabinoid matched Regex "%s".', original_cannabinoid_name, cannabinoid_regex.pattern)
  						if regex_matched:
  							# Match more than one regex?
  							logging.error('%s: Cannabinoid matches multiple patterns: %s (at list index %d).', raw_sample_file_name, cannabinoid_name, i)
  							skip_this_file = True
  						else:
  							logging.debug('%s: Regex matched first time: "%s".', raw_sample_file_name, cannabinoid_regex.pattern)
  							regex_matched = True
  						if cannabinoid_name in cannabinoid_data:
  							# Multiple items match same regex
  							logging.error('%s: Cannabinoid already recorded: %s (at list index %d).', raw_sample_file_name, cannabinoid_name, i)
  							skip_this_file = True
  						else:
  							cannabinoid_data[cannabinoid_name] = cannabinoid_amount
  
  				if original_cannabinoid_name is None:
  					logging.warning('%s: Cannabinoid name empty (at list index %d).', raw_sample_file_name, i)
  				elif not regex_matched:
  					# Match none?
  					logging.error('%s: Cannabinoid did not match anything: %s (at list index %d).', raw_sample_file_name, original_cannabinoid_name, i)
  			if cannabinoid_data == {}:
  				logging.debug('%s: No cannabinoids were added (%d were found).', raw_sample_file_name, len(raw_cannabinoids_1))
  		if args.force_cannabinoids and cannabinoid_data == {}:
  			skip_this_file = True
  
  		# 2 Test Data THC Total
  		thc_total = None
  		raw_thc_total = get_single_value(
  			tree=tree,
  			xpath=xpath_thc_total,
  			fallback='',
  			errlevel=logging.ERROR,
  			errmsg='%s: THC TOTAL was not found',
  			errparams=[raw_sample_file_name]
  		)
  		thc_amount_match = re_percentageValue.match(raw_thc_total)
  		if thc_amount_match:
  			try:
  				thc_total = normalize_number(
  					numberstring=raw_thc_total
  				)
  			except ValueError as e:
  				logging.warning('%s: THC-Total number error: %s.', raw_sample_file_name, raw_thc_total)
  		else:
  			logging.info('%s: THC-Total not percentage: %s.', raw_sample_file_name, raw_thc_total)
  
  		# 3 Test Data CBD Total
  		cbd_total = None
  		raw_cbd_total = get_single_value(
  			tree=tree,
  			xpath=xpath_cbd_total,
  			fallback='',
  			errlevel=logging.ERROR,
  			errmsg='%s: CBD TOTAL was not found',
  			errparams=[raw_sample_file_name]
  		)
  		cbd_amount_match = re_percentageValue.match(raw_cbd_total)
  		if cbd_amount_match:
  			try:
  				cbd_total = normalize_number(
  					numberstring=raw_cbd_total
  				)
  			except ValueError as e:
  				logging.warning('%s: CBD-Total number error: %s.', raw_sample_file_name, raw_cbd_total)
  		else:
  			logging.info('%s: CBD-Total not percentage: %s.', raw_sample_file_name, raw_cbd_total)
  
  		# 2 Sample Type
  		sample_type = None
  		raw_sample_type = get_single_value(
  			tree=tree,
  			xpath=xpath_sample_type,
  			fallback='',
  			errlevel=logging.ERROR,
  			errmsg='%s: Sample type was not found',
  			errparams=[raw_sample_file_name]
  		)
  		if raw_sample_type:
  			regex_matched = False
  			for sampletype_name in sample_types.keys():
  				sampletype_regex = sample_types[sampletype_name]
  				logging.debug('%s: Trying Regex "%s".', raw_sample_file_name, sampletype_regex)
  				sampletype_match = sampletype_regex.match(raw_sample_type)
  				if sampletype_match:
  					logging.debug('%s: Sample type matched Regex "%s".', raw_sample_type, sampletype_regex.pattern)
  					if regex_matched:
  						# Match more than one regex?
  						logging.error('%s: Sample type Regex "%s" matched before (sample type is now %s).', raw_sample_file_name, sampletype_regex.pattern, sampletype_name)
  						skip_this_file = True
  					else:
  						logging.debug('%s: Regex matched first time: "%s".', raw_sample_file_name, sampletype_regex.pattern)
  						regex_matched = True
  						sample_type = sampletype_name
  		# if not os.path.exists(os.path.join(args.database, sample_type)):
  		# 	os.makedirs(os.path.join(os.path.expanduser(args.database),sample_type), exist_ok=True)
  		if not regex_matched:
  			# Match none?
  			logging.error('%s: Sample type "%s" did not match anything (Xpath was %s).', raw_sample_file_name, raw_sample_type, xpath_canonicalURL)
  
  		# 3 Sample Name
  		sample_name = get_single_value(
  			tree=tree,
  			xpath=xpath_sample_name,
  			errlevel=logging.error,
  			errmsg='%s: Sample name was not found',
  			errparams=[raw_sample_file_name]
  		)
  
  		# 4 Sample Provider
  		sample_provider = get_single_value(
  			tree=tree,
  			xpath=xpath_sample_provider,
  			fallback=get_single_value(
  				tree=tree,
  				xpath=xpath_sample_provider_anon,
  				fallback='',
  				errlevel=logging.ERROR,
  				errmsg='%s: Provider was not found',
  				errparams=[raw_sample_file_name]
  			)
  		)
  		if sample_provider == 'Anonymous':
  			sample_provider = None
  		elif sample_provider is not None:
  			if sample_provider not in providers:
  				providers.append(sample_provider)
  			sample_provider = str(providers.index(sample_provider) + 1)
  
  		# 5 Test UID
  		test_uid = None
  		raw_test_uid = get_single_value(
  			tree=tree,
  			xpath=xpath_test_uid,
  			fallback='',
  			errlevel=logging.ERROR,
  			errmsg='%s: Test UID was not found',
  			errparams=[raw_sample_file_name]
  		)
  		test_uid_match = re_test_uid.match(raw_test_uid)
  		if test_uid_match:
  			test_uid = test_uid_match.group('uid')
  
  		# 6 Test Time
  		test_time = None
  		raw_test_time = get_single_value(
  			tree=tree,
  			xpath=xpath_time_tested,
  			fallback=''
  		)
  		re_date_match = re_date.match(raw_test_time)
  		if re_date_match:
  			possible_dates = dateparser_search.search_dates(
  				text=raw_test_time,
  				languages=['en'],
  				settings={'DATE_ORDER':'MDY','STRICT_PARSING':True}
  			)
  			if type(possible_dates) == list and len(possible_dates) == 1:
  				test_time = possible_dates[0][1].date().isoformat()
  			else:
  				logging.warning('%s: Test date (%s) was not understood.', raw_sample_file_name, raw_test_time)
  		else:
  			logging.error('%s: No test date was found (%s).', raw_sample_file_name, raw_test_time)
  
  		# 7 Receipt Time
  		receipt_time = None
  		raw_receipt_time = get_single_value(
  			tree=tree,
  			xpath=xpath_time_received,
  			fallback=''
  		)
  		re_date_match = re_date.match(raw_receipt_time)
  		if re_date_match:
  			possible_dates = dateparser_search.search_dates(
  				text=raw_receipt_time,
  				languages=['en'],
  				settings={'DATE_ORDER':'MDY','STRICT_PARSING':True}
  			)
  			if type(possible_dates) == list and len(possible_dates) == 1:
  				receipt_time = possible_dates[0][1].date().isoformat()
  			else:
  				logging.warning('%s: Receipt date (%s) was not understood.', raw_sample_file_name, raw_receipt_time)
  		else:
  			logging.error('%s: No receipt date was found (%s).', raw_sample_file_name, raw_receipt_time)
  
  		if terpenes_data == {} and cannabinoid_data == {}:
  			skip_this_file = True
  
  		sample_data = {
  			'Test Result UID':test_uid,
  			'Sample Name':sample_name,
  			'Test Time':test_time,
  			'Receipt Time':receipt_time,
  			'Provider':sample_provider,
  			'Terpene TOTAL':terpene_total,
  			'THC TOTAL':thc_total,
  			'CBD TOTAL':cbd_total,
  			'Sample Type':sample_type
  		}
  		sample_data.update(terpenes_data)
  		sample_data.update(cannabinoid_data)
  
  		for sample_data_field in list(sample_data.keys()):
  			remove_key = False
  			if sample_data[sample_data_field] is None:
  				remove_key = True
  			if sample_data_field not in DATA_ROW_FIELDS:
  				remove_key = True
  			if remove_key:
  				del sample_data[sample_data_field]
  		if sample_data == {}:
  			skip_this_file = True
  
  		if not skip_this_file:
  			if args.csv:
  				write_to_csv(
  					filepath=sample_database_CSVfile,
  					fieldnames=DATA_ROW_FIELDS,
  					data=sample_data
  				)
  			if args.json:
  				with open(sample_database_JSONfile, "a", encoding="utf-8") as databases_file:
  					if not is_first_sample:
  						databases_file.write(',')
  					json.dump(sample_data, databases_file, separators=(',', ':'), sort_keys=True)
  					is_first_sample = False
  	if args.json:
  		with open(sample_database_JSONfile, "a", encoding="utf-8") as databases_file:
  			databases_file.write(']')
  
  logging.debug('Finished main loop.')
  
  if args.json:
  	with open(sample_database_JSONfile, "a", encoding="utf-8") as databases_file:
  		databases_file.write('}}')
  
  print('All files have been processed. If you defined a log file, please check its contents. It lists pages regarding different errors.')
  