text: import logging
  import time
  from importers.common import taxonomy
  from elasticsearch.exceptions import RequestError
  
  from importers.common.format_conversions import is_ad_remote
  from importers.common.helpers import clean_html, isodate, get_null_safe_value
  from importers.common.keywords_extracted import add_keywords_extracted
  
  logging.basicConfig()
  logging.getLogger(__name__).setLevel(logging.INFO)
  log = logging.getLogger(__name__)
  
  MUST_HAVE_WEIGHT = 10
  NICE_TO_HAVE_WEIGHT = 5
  
  
  def convert_ad(message):
      annons = dict()
      start_time = int(time.time() * 1000)
      if 'removed_ad_filter' in message:
          annons['removed_ad_filter'] = message.get('removed_ad_filter')
  
      annons['id'] = message.get('annonsId')
      annons['external_id'] = message.get('externtAnnonsId')
      annons['headline'] = message.get('annonsrubrik')
      annons['application_deadline'] = isodate(message.get('sistaAnsokningsdatum'))
      annons['number_of_vacancies'] = message.get('antalPlatser')
  
      cleaned_description_text = clean_html(message.get('annonstextFormaterad'))
      if cleaned_description_text == '' and not message.get('avpublicerad'):
          log.warning(f"description.text is empty for ad id: {annons['id']}")
  
      if not message.get('avpublicerad'):
          annons['remote_work'] = is_ad_remote(cleaned_description_text, annons['headline'])
  
      annons['description'] = {
          'text': cleaned_description_text,
          'text_formatted': message.get('annonstextFormaterad'),
          'company_information': message.get('ftgInfo'),
          'needs': message.get('beskrivningBehov'),
          'requirements': message.get('beskrivningKrav'),
          'conditions': message.get('villkorsbeskrivning'),
      }
      annons['employment_type'] = _expand_taxonomy_value_and_replaced('anstallningstyp', 'anstallningTyp', message)
      annons['salary_type'] = _expand_taxonomy_value('lonetyp', 'lonTyp', message)
      annons['salary_description'] = message.get('lonebeskrivning')
      annons['duration'] = _expand_taxonomy_value('varaktighet', 'varaktighetTyp', message)
      annons['working_hours_type'] = _expand_taxonomy_value('arbetstidstyp', 'arbetstidTyp', message)
      (default_min_omf, default_max_omf) = _get_default_scope_of_work(message.get('arbetstidTyp', {}).get('varde'))
      annons['scope_of_work'] = {
          'min': message.get('arbetstidOmfattningFran', default_min_omf),
          'max': message.get('arbetstidOmfattningTill', default_max_omf)
      }
      annons['access'] = message.get('tilltrade')
      annons['employer'] = {
          'phone_number': message.get('telefonnummer'),
          'email': message.get('epost'),
          'url': message.get('webbadress'),
          'organization_number': message.get('organisationsnummer'),
          'name': message.get('arbetsgivareNamn'),
          'workplace': message.get('arbetsplatsNamn'),
          'workplace_id': message.get('arbetsplatsId')
      }
      annons['application_details'] = {
          'information': message.get('informationAnsokningssatt'),
          'reference': message.get('referens'),
          'email': message.get('ansokningssattEpost'),
          'via_af': message.get('ansokningssattViaAF'),
          'url': message.get('ansokningssattWebbadress'),
          'other': message.get('ansokningssattAnnatSatt')
      }
      annons['experience_required'] = not message.get('ingenErfarenhetKravs', False)
      annons['access_to_own_car'] = message.get('tillgangTillEgenBil', False)
      if message.get('korkort', []):
          annons['driving_license_required'] = True
          annons['driving_license'] = parse_driving_licence(message)
      else:
          annons['driving_license_required'] = False
  
      _set_occupations(annons, message)
  
      annons['workplace_address'] = _build_wp_address(message.get('arbetsplatsadress', {}))
  
      annons['must_have'] = {
          'skills': [skill for kompetens in message.get('kompetenser', [])
                     for skill in _get_concept_and_replaced_with_weight(
                  'kompetens', kompetens.get('varde'), kompetens.get('vikt'))
                     if get_null_safe_value(kompetens, 'vikt', 0) >= MUST_HAVE_WEIGHT
                     ],
          'languages': [
              _get_concept_as_annons_value_with_weight('sprak', sprak.get('varde'),
                                                       sprak.get('vikt'))
              for sprak in message.get('sprak', [])
              if get_null_safe_value(sprak, 'vikt', 0) >= MUST_HAVE_WEIGHT
          ],
          'work_experiences': [
              _get_concept_as_annons_value_with_weight('yrkesroll', yrkerf.get('varde'), yrkerf.get('vikt'))
              for yrkerf in message.get('yrkeserfarenheter', [])
              if get_null_safe_value(yrkerf, 'vikt', 0) >= MUST_HAVE_WEIGHT
          ],
          'education': [],
          'education_level': [],
      }
  
      annons['nice_to_have'] = {
          'skills': [skill for kompetens in message.get('kompetenser', [])
                     for skill in _get_concept_and_replaced_with_weight(
                  'kompetens', kompetens.get('varde'), kompetens.get('vikt'))
                     if get_null_safe_value(kompetens, 'vikt', NICE_TO_HAVE_WEIGHT) < MUST_HAVE_WEIGHT
                     ],
          'languages': [
              _get_concept_as_annons_value_with_weight('sprak', sprak.get('varde'), sprak.get('vikt'))
              for sprak in message.get('sprak', [])
              if get_null_safe_value(sprak, 'vikt', NICE_TO_HAVE_WEIGHT) < MUST_HAVE_WEIGHT
          ],
          'work_experiences': [
              _get_concept_as_annons_value_with_weight('yrkesroll', yrkerf.get('varde'), yrkerf.get('vikt'))
              for yrkerf in message.get('yrkeserfarenheter', [])
              if get_null_safe_value(yrkerf, 'vikt', NICE_TO_HAVE_WEIGHT) < MUST_HAVE_WEIGHT
          ],
          'education': [],
          'education_level': [],
      }
  
      annons['application_contact'] = build_contacts(message.get('kontaktpersoner', []), annons['id'])
  
      if message.get('utbildningsinriktning'):
          try:
              _set_education(annons, message)
          except TypeError as e:
              log.warning(f"Skipping education on ad: {annons['id']} due to TypeError: {e}")
  
      annons['publication_date'] = isodate(message.get('publiceringsdatum'))
      annons['last_publication_date'] = isodate(message.get('sistaPubliceringsdatum'))
      annons['removed'] = message.get('avpublicerad')
      annons['removed_date'] = isodate(message.get('avpubliceringsdatum'))
      annons['source_type'] = message.get('kallaTyp')
      annons['timestamp'] = message.get('updatedAt')
      annons['logo_url'] = message.get('logo_url')
      # Extract labels as keywords for easier searching
      log.debug(f"Convert ad: {annons['id']} took: {int(time.time() * 1000) - start_time} ms")
      return add_keywords_extracted(annons)
  
  
  def _set_education(annons, message):
      if get_null_safe_value(message.get('utbildningsinriktning'), 'vikt', 0) >= MUST_HAVE_WEIGHT:
          annons['must_have']['education'] = [
              _get_concept_as_annons_value_with_weight(
                  ['sun-education-field-1', 'sun-education-field-2', 'sun-education-field-3'],
                  message.get('utbildningsinriktning', {}).get('varde'),
                  message.get('utbildningsinriktning', {}).get('vikt'))]
          annons['must_have']['education_level'] = [
              _get_concept_as_annons_value_with_weight(
                  ['sun-education-level-1', 'sun-education-level-2', 'sun-education-level-3'],
                  message.get('utbildningsniva', {}).get('varde'),
                  message.get('utbildningsniva', {}).get('vikt'))]
      else:
          annons['nice_to_have']['education'] = [
              _get_concept_as_annons_value_with_weight(
                  ['sun-education-field-1', 'sun-education-field-2', 'sun-education-field-3'],
                  message.get('utbildningsinriktning', {}).get('varde'),
                  message.get('utbildningsinriktning', {}).get('vikt'))]
          annons['nice_to_have']['education_level'] = [
              _get_concept_as_annons_value_with_weight(
                  ['sun-education-level-1', 'sun-education-level-2', 'sun-education-level-3'],
                  message.get('utbildningsniva', {}).get('varde'),
                  message.get('utbildningsniva', {}).get('vikt'))]
  
  
  # TODO: unit tests, does it handle new concept ids?
  def _get_default_scope_of_work(arbtid_typ):
      default_max_omf = None
      default_min_omf = None
  
      # If heltid...
      if arbtid_typ == "6YE1_gAC_R2G":
          default_min_omf = 100
          default_max_omf = 100
      # Elif deltid or heltid/deltid
      elif arbtid_typ == "947z_JGS_Uk2" or arbtid_typ == "hGCt_6Ni_XPz":
          default_min_omf = 0
          default_max_omf = 100
      elif arbtid_typ:
          log.warning(f"Unknown working_hours_type (arbetstidTyp): {arbtid_typ}")
  
      return default_min_omf, default_max_omf
  
  
  def is_union_representative(kontaktperson):
      """
      if a contact person is a union representative can be determined in two ways:
      1. boolean value for field 'fackligRepresentant'
      2. text 'facklig representant in field 'beskrivning'
      """
      if kontaktperson.get('fackligRepresentant'):
          return True
  
      # check all fields for matching string just to be sure, even if 'fackligRepresentant' is not True
      check_for = 'facklig representant'
      for key, value in kontaktperson.items():
          if value and not isinstance(value, bool):
              if check_for in value.lower():
                  return True
      return False
  
  
  def build_contacts(kontaktpersoner, ad_id='1'):
      appl_contact_list = []
      for kontaktperson in kontaktpersoner:
          if not is_union_representative(kontaktperson):
              appl_contact = {}
              name = f"{get_null_safe_value(kontaktperson, 'fornamn', '')} {get_null_safe_value(kontaktperson, 'efternamn', '')}"
              appl_contact['name'] = name.strip() or None
              appl_contact['description'] = kontaktperson.get('beskrivning')
              appl_contact['email'] = kontaktperson.get('epost')
              phone_numbers = f"{get_null_safe_value(kontaktperson, 'telefonnummer', '')}, {get_null_safe_value(kontaktperson, 'mobilnummer', '')}"
              appl_contact['telephone'] = phone_numbers.strip(', ') or None
              appl_contact['contactType'] = kontaktperson.get('befattning')
              appl_contact_list.append(appl_contact)
          else:
              log.debug(f'Found union person in id: {ad_id}')
  
      return appl_contact_list
  
  
  # TODO return value so that it can be tested easily
  def _set_occupations(annons, message):
      if 'yrkesroll' in message:
          yrkesroll = taxonomy.get_legacy_by_concept_id('yrkesroll', message.get('yrkesroll', {}).get('varde'))
  
          if yrkesroll and 'parent' in yrkesroll:
              yrkesgrupp = yrkesroll.get('parent', {})
              yrkesomrade = yrkesgrupp.get('parent', {})
              yrkesrolls_replaced = yrkesroll.get('replaced_by', [])
              collections = yrkesroll.get('collections', {})
              annons['occupation'] = [{'concept_id': yrkesroll.get('concept_id', None),
                                       'label': message.get('yrkesroll', {}).get('namn'),
                                       'legacy_ams_taxonomy_id': yrkesroll.get('legacy_ams_taxonomy_id', None),
                                       'original_value': True}]
              # changed occupation name while same concept_id
              if yrkesroll.get('label', None) != message.get('yrkesroll', {}).get('namn'):
                  annons['occupation'] += [{'concept_id': yrkesroll.get('concept_id', None),
                                            'label': yrkesroll.get('label', None),
                                            'legacy_ams_taxonomy_id': yrkesroll.get('legacy_ams_taxonomy_id', None)}]
              if yrkesrolls_replaced:
                  annons['occupation'] += [{'concept_id': yrkesroll_replaced.get('concept_id', None),
                                            'label': yrkesroll_replaced.get('label', None),
                                            'legacy_ams_taxonomy_id': yrkesroll_replaced.get('legacy_ams_taxonomy_id',
                                                                                             None)}
                                           for yrkesroll_replaced in yrkesrolls_replaced]
              annons['occupation_group'] = [{'concept_id': yrkesgrupp.get('concept_id', None),
                                             'label': yrkesgrupp.get('label', None),
                                             'legacy_ams_taxonomy_id':
                                                 yrkesgrupp.get('legacy_ams_taxonomy_id', None)}]
              annons['occupation_field'] = [{'concept_id': yrkesomrade.get('concept_id', None),
                                             'label': yrkesomrade.get('label', None),
                                             'legacy_ams_taxonomy_id':
                                                 yrkesomrade.get('legacy_ams_taxonomy_id', None)}]
              annons['collections'] = [{'concept_id': collection.get('concept_id', None),
                                        'label': collection.get('label', None),
                                        'legacy_ams_taxonomy_id': collection.get('legacy_ams_taxonomy_id', None)}
                                       for collection in collections]
          elif not yrkesroll:
              log.warning(f"Taxonomy value not found for: {message['yrkesroll']}, ad: {annons['id']}")
          else:  # yrkesroll is not None and 'parent' not in yrkesroll
              log.warning(f"Parent not found for yrkesroll: {message['yrkesroll']}, ({yrkesroll}), ad: {annons['id']}")
      else:
          log.warning(f"Word 'yrkesroll' is not found in: {message}, ad: {annons['id']}")
  
  
  def _build_wp_address(arbplats_message):
      kommun = None
      lansnamn = None
      kommun_concept_id = None
      kommun_legacy_id = None
      land = None
      lan_legacy_id = None
      lan_concept_id = None
      land_legacy_id = None
      land_concept_id = None
      longitud = None
      latitud = None
      if 'kommun' in arbplats_message and arbplats_message.get('kommun'):
          kommun_concept_id = arbplats_message.get('kommun', {}).get('varde', {})
          kommun_temp = taxonomy.get_legacy_by_concept_id('kommun', kommun_concept_id)
          kommun_legacy_id = kommun_temp.get('legacy_ams_taxonomy_id', None)
          kommun = arbplats_message.get('kommun', {}).get('namn', {})
      if 'lan' in arbplats_message and arbplats_message.get('lan'):
          lan_concept_id = arbplats_message.get('lan', {}).get('varde', {})
          lan_temp = taxonomy.get_legacy_by_concept_id('lan', lan_concept_id)
          lan_legacy_id = lan_temp.get('legacy_ams_taxonomy_id', None)
          lansnamn = arbplats_message.get('lan', {}).get('namn', {})
      if 'land' in arbplats_message and arbplats_message.get('land'):
          land_concept_id = arbplats_message.get('land', {}).get('varde', {})
          land_temp = taxonomy.get_legacy_by_concept_id('land', land_concept_id)
          land_legacy_id = land_temp.get('legacy_ams_taxonomy_id', None)
          land = arbplats_message.get('land', {}).get('namn', {})
      if 'longitud' in arbplats_message and arbplats_message.get('longitud'):
          longitud = float(arbplats_message.get('longitud'))
      if 'latitud' in arbplats_message and arbplats_message.get('latitud'):
          latitud = float(arbplats_message.get('latitud'))
  
      return {
          'municipality_code': kommun_legacy_id,
          'municipality_concept_id': kommun_concept_id,
          'municipality': kommun,
          'region_code': lan_legacy_id,
          'region_concept_id': lan_concept_id,
          'region': lansnamn,
          'country_code': land_legacy_id or "199",
          'country_concept_id': land_concept_id or "i46j_HmG_v64",
          'country': land or "Sverige",
          # 'street_address': message.get('besoksadress', {}).get('gatuadress'),
          'street_address': arbplats_message.get('gatuadress', ''),
          # 'postcode': message.get('postadress', {}).get('postnr'),
          'postcode': arbplats_message.get('postnr'),
          # 'city': message.get('postadress', {}).get('postort'),
          'city': arbplats_message.get('postort'),
          'coordinates': [longitud, latitud]
      }
  
  
  def _expand_taxonomy_value(annons_key, message_key, message_dict):
      message_value = message_dict.get(message_key, {}).get('varde') if message_dict else None
      if message_value:
          if concept := taxonomy.get_legacy_by_concept_id(annons_key, message_value):
              return {
                  "concept_id": concept.get('concept_id', None),
                  "label": concept.get('label', None),
                  "legacy_ams_taxonomy_id": concept.get('legacy_ams_taxonomy_id', None)
              }
          else:
              log.debug(f"(expand_taxonomy_value)Empty message_key: {message_value} Type: {annons_key} Key:{message_key},"
                        f" Id: {message_dict.get('annonsId') if message_dict else None}")
              return None
      else:
          ads_id = message_dict.get('annonsId') if message_dict else None
          log.debug(
              f"(expand_taxonomy_value)Empty message_key: {message_value} Type: {annons_key} Key:{message_key} Id: {ads_id} ")
      return None
  
  
  def _expand_taxonomy_value_and_replaced(annons_key, message_key, message_dict):
      message_value = message_dict.get(message_key, {}).get('varde') if message_dict else None
      values = []
      if message_value:
          concept = taxonomy.get_legacy_by_concept_id(annons_key, message_value)
          if concept:
              values.append({
                  "concept_id": concept.get('concept_id', None),
                  "label": concept.get('label', None),
                  "legacy_ams_taxonomy_id": concept.get('legacy_ams_taxonomy_id', None),
                  "original_value": True
              })
              replaced_values = concept.get('replaced_by', [])
              if replaced_values:
                  values += [{"concept_id": replaced_value.get('concept_id', None),
                              "label": replaced_value.get('label', None),
                              "legacy_ams_taxonomy_id": replaced_value.get('legacy_ams_taxonomy_id', None)}
                             for replaced_value in replaced_values]
      return values
  
  
  def _get_concept_as_annons_value_with_weight(taxtype, id, weight=None):
      weighted_concept = {
          'concept_id': None,
          'label': None,
          'weight': None,
          'legacy_ams_taxonomy_id': None
      }
      try:
          concept = taxonomy.get_legacy_by_concept_id(taxtype, id)
          weighted_concept['concept_id'] = concept.get('concept_id', None)
          weighted_concept['label'] = concept.get('label', None)
          weighted_concept['weight'] = weight
          weighted_concept['legacy_ams_taxonomy_id'] = concept.get('legacy_ams_taxonomy_id', None)
      except AttributeError:
          log.warning(f'Taxonomy (3) value not found for {taxtype} {id}')
      except RequestError:
          log.warning(f'Request failed with argtype: {taxtype} and legacy_id or concept_id: {id}')
      return weighted_concept
  
  
  def _get_concept_and_replaced_with_weight(tax_type, id, weight=None):
      concepts = []
      weighted_concept = {
          'concept_id': None,
          'label': None,
          'weight': None,
          'legacy_ams_taxonomy_id': None,
      }
      try:
          concept = taxonomy.get_legacy_by_concept_id(tax_type, id)
          weighted_concept['concept_id'] = concept.get('concept_id', None)
          weighted_concept['label'] = concept.get('label', None)
          weighted_concept['weight'] = weight
          weighted_concept['legacy_ams_taxonomy_id'] = concept.get('legacy_ams_taxonomy_id', None)
          weighted_concept['original_value'] = True
          concepts.append(weighted_concept)
          replaced_concepts = concept.get('replaced_by', [])
          if replaced_concepts:
              concepts += [{'concept_id': replaced_concept.get('concept_id', None),
                            'label': replaced_concept.get('label', None),
                            'weight': weight,
                            'legacy_ams_taxonomy_id': replaced_concept.get('label', None)}
                           for replaced_concept in replaced_concepts]
      except AttributeError:
          log.warning(f'Taxonomy value not found for type: {tax_type}, id: {id}')
      except RequestError:
          log.warning(f'Request failed with argtype: {tax_type} and legacy_id or concept_id: {id}')
      return concepts
  
  
  def parse_driving_licence(message):
      tax_korkort_list = []
      for korkort in message.get('korkort'):
          tax_korkort = taxonomy.get_legacy_by_concept_id('korkort', korkort['varde'])
          if tax_korkort:
              tax_korkort_list.append({
                  "concept_id": tax_korkort.get('concept_id', None),
                  "legacy_ams_taxonomy_id": tax_korkort.get('legacy_ams_taxonomy_id', None),
                  "label": tax_korkort.get('label', None)
              })
      return tax_korkort_list
  