text: #!/usr/bin/env python3
  
  import configparser as ConfigParser
  import io
  import json
  import logging
  import os
  from pathlib import Path
  from fuji_server.controllers.fair_check import FAIRCheck
  from fuji_server.helper.preprocessor import Preprocessor
  import gc
  import tracemalloc
  # os
  #os.environ['TIKA_SERVER_JAR'] = 'file://Program Files\tika\tika-server-1.24.1.jar'
  
  identifier = 'https://doi.org/10.1594/PANGAEA.902845'
  #identifier='https://doi.org/10.26050/WDCC/MOMERGOMBSCMAQ'
  oai_pmh = 'http://ws.pangaea.de/oai/'
  debug = True
  
  muchotestpids=[
      '10.15493/DEFF.10000003','https://phaidra.cab.unipd.it/view/o:267291',
      'doi:10.1038/nphys1170','doi:10.17882/42182','https://deims.org/sites/default/files/data/elter_va_fruska_gora_temperature_0.xls',
      '10.25504/FAIRsharing.2bdvmk','http://bio2rdf.org/affymetrix:1415765_at','doi:10.18129/B9.bioc.BiocGenerics',
      'https://data.noaa.gov/dataset/dataset/w00411-nos-hydrographic-survey-2015-08-15','10.6075/J0513WJD','10.7280/D1P075',
      '10.1007/s10531-013-0468-6','https://neurovault.org/images/13953/','10.17605/OSF.IO/XFWS6','https://hdl.handle.net/10411/G8MPEI',
      '10.17870/bathspa.7926890.v1','http://thredds.met.no/thredds/catalog/met.no/observations/stations/catalog.html?dataset=met.no/observations/stations/SN999',
      'https://hdl.handle.net/11676/Hz8P-d-sstjMXCmWGTY67a2O','10.24435/materialscloud:2019.0013/v2','https://www.gbif.org/dataset/4a65dba1-ff0d-4e72-aa3c-e76e48856930',
      'hdl:10037.1/10152','https://repo.clarino.uib.no/xmlui/handle/11509/95','https://hunt-db.medisin.ntnu.no/hunt-db/#/studypart/432',
      'https://www.proteinatlas.org/ENSG000002695','http://gis.ices.dk/geonetwork/srv/eng/catalog.search#/metadata/33fa648d-c4d6-4449-ac3c-dbec0f204e1d',
      'https://data.geus.dk/JupiterWWW/anlaeg.jsp?anlaegid=97389','http://tun.fi/JX.1058739','10.15468/0fxsox',
      'https://doi.pangaea.de/10.1594/PANGAEA.866933','10.17026/dans-z56-fz75','http://data.europa.eu/89h/jrc-eplca-898618b5-3306-11dd-bd11-0800200c9a66',
      'https://www.govdata.de/web/guest/suchen/-/details/temperatur-des-meerwassers-20172b0eb','doi:10.26050/WDCC/MOMERGOMBSCMAQ',
      'https://lithologs.net/?action=view&oid=470','http://bio2rdf.org/affymetrix:1415765_at','https://www.data.gv.at/katalog/dataset/b8dac7af-5c8a-4936-9abe-e1dbbdd8dd4f',
      'https://identifiers.org/ena.embl:BN000065','https://www.ebi.ac.uk/biosamples/samples/SAMN14168013','https://data.gov.lv/dati/lv/dataset/covid-19',
      'https://www.data.gouv.fr/datasets/5e7e104ace2080d9162b61d8','https://www.data.gv.at/katalog/dataset/b8dac7af-5c8a-4936-9abe-e1dbbdd8dd4f',
      'https://datos.gob.es/es/catalogo/e05070101-evolucion-de-enfermedad-por-el-coronavirus-covid-19','https://doi.org/10.26050/WDCC/MOMERGOMBSCMAQ',
      'https://data.gov.lv/dati/lv/dataset/covid-19','https://raw.githubusercontent.com/obi-ontology/obi/v2020-04-23/obi.owl','http://vocab.nerc.ac.uk/collection/L22/current/',
      'http://purl.org/vocommons/voaf','http://purl.obolibrary.org/obo/bfo.owl','http://purl.obolibrary.org/obo/bfo.owl',
      'https://raw.githubusercontent.com/BFO-ontology/BFO/v2019-08-26/bfo_classes_only.owl','http://vocab.nerc.ac.uk/collection/L05/current/',
      'https://doi.org/10.4232/1.13491','http://dx.doi.org/10.18712/NSD-NSD2202-2-v1','http://doi.org/10.5255/UKDA-SN-854270', 'https://doi.org/10.1594/PANGAEA.896660',
      'http://sweetontology.net/matrRockIgneous','http://hdl.handle.net/20.500.12124/7','http://hdl.handle.net/11495/DAB8-B44B-E8C8-2','https://doi.pangaea.de/10.1594/PANGAEA.919306',
      'https://doi.org/10.7302/Z24Q7S64','10.5676/DWD_GPCC/MP_M_V6_100','10.6092/84cb588d-97e5-4c64-91bb-ba6109dfa530',
      'https://doi.org/10.17863/CAM.14473','doi:10.5441/001/1.44cb3946','https://doi.org/10.25704/Z5Y2-QPYE','https://cera-www.dkrz.de/WDCC/ui/cerasearch/entry?acronym=CMAQ_t_det_w_atmos_tot_N_17',
      'http://doi.org/10.25914/5eaa30de53244','http://dx.doi.org/10.4227/05/5344F1159A1A9','https://deims.org/dataset/75a7f938-7c77-11e3-8832-005056ab003f',
      'https://hdl.handle.net/11676/Ok72Hfm8xlJkhg-9lOQpsZGr','https://data.neonscience.org/data-products/DP1.20066.001','https://doi.pangaea.de/10.1594/PANGAEA.920063',
      'https://doi.pangaea.de/10.1594/PANGAEA.896543','https://dx.doi.org/10.4227/05/5344F1159A1A9','https://doi.org/10.4225/08/563869A931CFE',
      'https://data.neonscience.org/data-products/DP1.00001.001','https://dx.doi.org/10.11922/sciencedb.293','https://deims.org/dataset/75a7f938-7c77-11e3-8832-005056ab003f',
      'https://meta.icos-cp.eu/objects/8YwZj8CQEj87IuI9P6QkZiKX','http://doi.org/10.22033/ESGF/CMIP6.4397','http://doi.org/10.25914/5eaa30de53244',
      'http://dda.dk/catalogue/150','https://doi.org/10.1594/PANGAEA.902845','https://doi.org/10.1594/PANGAEA.745671','http://dda.dk/catalogue/150','https://ckan.govdata.de/ja/dataset/bebauungsplan-rahlstedt-131-hamburgb809f',
      'https://deims.org/dataset/75a7f938-7c77-11e3-8832-005056ab003f','https://hdl.handle.net/11168/11.429265',
      'https://metadata.bgs.ac.uk/geonetwork/srv/api/records/6abc401d-250a-5469-e054-002128a47908','https://data.dtu.dk/articles/Data_for_the_paper_A_dual_reporter_system_for_investigating_and_optimizing_translation_and_folding_in_E_coli_/10265420',
      'https://www.proteinatlas.org/ENSG00000110651-CD81/cell','http://dda.dk/catalogue/868','https://ckan.govdata.de/ja/dataset/bebauungsplan-rahlstedt-65-hamburg', 'https://ortus.rtu.lv/science/en/datamodule/294',
      'https://doi.org/10.15482/USDA.ADC/1324677','https://www.proteinatlas.org/ENSG00000110651-CD81/cell','http://doi.org/10.5255/UKDA-SN-1329-1',
      'http://purl.org/vocommons/voaf','https://meta.icos-cp.eu/objects/9ri1elaogsTv9LQFLNTfDNXm','http://doi.org/10.22033/ESGF/CMIP6.4397',
      'https://su.figshare.com/articles/Data_for_Does_historical_land_use_affect_the_regional_distribution_of_fleshy-fruited_woody_plants_Arnell_et_al_2019_/10318046','http://doi.org/10.1007/s10531-013-0468-6',
      'https://data.gov.lv/dati/lv/dataset/maksatnespejas-procesi','https://databank.ora.ox.ac.uk/UniversityCollege/datasets/04156fde-dabb-48fd-baf6-533182f74b5b'
  ]
  
  testpids=[
      '10.15493/DEFF.10000003',
      'doi:10.1038/nphys1170','doi:10.17882/42182','https://deims.org/sites/default/files/data/elter_va_fruska_gora_temperature_0.xls',
      '10.25504/FAIRsharing.2bdvmk','http://bio2rdf.org/affymetrix:1415765_at','doi:10.18129/B9.bioc.BiocGenerics',
      'https://data.noaa.gov/dataset/dataset/w00411-nos-hydrographic-survey-2015-08-15','10.6075/J0513WJD','10.7280/D1P075',
      '10.1007/s10531-013-0468-6','https://neurovault.org/images/13953/','10.17605/OSF.IO/XFWS6','https://hdl.handle.net/10411/G8MPEI',
      '10.17870/bathspa.7926890.v1','http://thredds.met.no/thredds/catalog/met.no/observations/stations/catalog.html?dataset=met.no/observations/stations/SN999',
      'https://hdl.handle.net/11676/Hz8P-d-sstjMXCmWGTY67a2O','10.24435/materialscloud:2019.0013/v2','https://www.gbif.org/dataset/4a65dba1-ff0d-4e72-aa3c-e76e48856930',
      'hdl:10037.1/10152','https://repo.clarino.uib.no/xmlui/handle/11509/95','https://hunt-db.medisin.ntnu.no/hunt-db/#/studypart/432',
      'https://www.proteinatlas.org/ENSG000002695','http://gis.ices.dk/geonetwork/srv/eng/catalog.search#/metadata/33fa648d-c4d6-4449-ac3c-dbec0f204e1d',
      'https://data.geus.dk/JupiterWWW/anlaeg.jsp?anlaegid=97389','http://tun.fi/JX.1058739','10.15468/0fxsox',
      'https://doi.pangaea.de/10.1594/PANGAEA.866933','10.17026/dans-z56-fz75','http://data.europa.eu/89h/jrc-eplca-898618b5-3306-11dd-bd11-0800200c9a66',
      'https://www.govdata.de/web/guest/suchen/-/details/temperatur-des-meerwassers-20172b0eb','doi:10.26050/WDCC/MOMERGOMBSCMAQ',
      'https://ortus.rtu.lv/science/en/datamodule/3'
  ]
  
  #testpids=['https://lithologs.net/?action=view&oid=470']
  # rdf links:
  #testpids=['http://bio2rdf.org/affymetrix:1415765_at','https://www.data.gv.at/katalog/dataset/b8dac7af-5c8a-4936-9abe-e1dbbdd8dd4f','https://identifiers.org/ena.embl:BN000065','https://www.ebi.ac.uk/biosamples/samples/SAMN14168013']
  #testpids=['https://data.gov.lv/dati/lv/dataset/covid-19','https://www.data.gouv.fr/datasets/5e7e104ace2080d9162b61d8','https://www.data.gv.at/katalog/dataset/b8dac7af-5c8a-4936-9abe-e1dbbdd8dd4f','https://datos.gob.es/es/catalogo/e05070101-evolucion-de-enfermedad-por-el-coronavirus-covid-19']
  #testpids=['https://doi.org/10.26050/WDCC/MOMERGOMBSCMAQ']
  #testpids=['https://data.gov.lv/dati/lv/dataset/covid-19']
  #ontologies
  #testpids=['https://raw.githubusercontent.com/obi-ontology/obi/v2020-04-23/obi.owl']
  testpids=['http://vocab.nerc.ac.uk/collection/L22/current/','http://purl.org/vocommons/voaf','http://purl.obolibrary.org/obo/bfo.owl']
  #testpids=['http://purl.obolibrary.org/obo/bfo.owl']
  #testpids=['https://raw.githubusercontent.com/BFO-ontology/BFO/v2019-08-26/bfo_classes_only.owl']
  #testpids=['http://vocab.nerc.ac.uk/collection/L05/current/']
  #social sciences
  #testpids=['https://doi.org/10.4232/1.13491','http://dx.doi.org/10.18712/NSD-NSD2202-2-v1','http://doi.org/10.5255/UKDA-SN-854270']
  #testpids=['https://doi.org/10.1594/PANGAEA.896660']
  #testpids=['http://sweetontology.net/matrRockIgneous']
  #clarin linguistics
  #testpids=['http://hdl.handle.net/20.500.12124/7','http://hdl.handle.net/11495/DAB8-B44B-E8C8-2']
  #restricted
  #testpids=['https://doi.pangaea.de/10.1594/PANGAEA.919306']
  #!!!!!!iana, ddi etc RDF graph, DOI not registered at datacite:
  #testpids=['https://doi.org/10.7302/Z24Q7S64']
  #testpids=['10.5676/DWD_GPCC/MP_M_V6_100']
  #testpids=['10.6092/84cb588d-97e5-4c64-91bb-ba6109dfa530']
  #FAIRsFAIR selected repositories
  #testpids=['https://doi.org/10.17863/CAM.14473','doi:10.5441/001/1.44cb3946','https://doi.org/10.25704/Z5Y2-QPYE']
  #testpids=['https://cera-www.dkrz.de/WDCC/ui/cerasearch/entry?acronym=CMAQ_t_det_w_atmos_tot_N_17']
  #testpids=['http://doi.org/10.25914/5eaa30de53244']
  # EML by guessed XML
  #testpids=['http://dx.doi.org/10.4227/05/5344F1159A1A9']
  #testpids=['https://deims.org/dataset/75a7f938-7c77-11e3-8832-005056ab003f']
  #testpids=['https://hdl.handle.net/11676/Ok72Hfm8xlJkhg-9lOQpsZGr']
  #testpids=['https://data.neonscience.org/data-products/DP1.20066.001']
  #not found
  #testpids=['https://doi.pangaea.de/10.1594/PANGAEA.920063']
  '''
  testpids=['https://doi.pangaea.de/10.1594/PANGAEA.896543',
  'https://dx.doi.org/10.4227/05/5344F1159A1A9',
  'https://doi.org/10.4225/08/563869A931CFE',
  'https://data.neonscience.org/data-products/DP1.00001.001',
  'https://dx.doi.org/10.11922/sciencedb.293',
  'https://deims.org/dataset/75a7f938-7c77-11e3-8832-005056ab003f',
  'https://meta.icos-cp.eu/objects/8YwZj8CQEj87IuI9P6QkZiKX',
  'http://doi.org/10.22033/ESGF/CMIP6.4397',
  'http://doi.org/10.25914/5eaa30de53244']
  '''
  #DCAT DDI
  #testpids=['http://dda.dk/catalogue/150']
  #very large file!!
  #testpids=['https://doi.org/10.1594/PANGAEA.902845']
  #testpids=['https://doi.org/10.1594/PANGAEA.745671']
  #testpids=['http://dda.dk/catalogue/150']
  #perfect DCAT
  #testpids=['https://ckan.govdata.de/ja/dataset/bebauungsplan-rahlstedt-131-hamburgb809f']
  #testpids=['https://doi.org/10.1594/PANGAEA.879324']
  #testpids=['https://deims.org/dataset/75a7f938-7c77-11e3-8832-005056ab003f','https://hdl.handle.net/11168/11.429265']
  #testpids=['https://metadata.bgs.ac.uk/geonetwork/srv/api/records/6abc401d-250a-5469-e054-002128a47908']
  #testpids=['https://data.dtu.dk/articles/Data_for_the_paper_A_dual_reporter_system_for_investigating_and_optimizing_translation_and_folding_in_E_coli_/10265420']
  #testpids=['https://www.proteinatlas.org/ENSG00000110651-CD81/cell']
  #testpids=['http://dda.dk/catalogue/868']
  #testpids=['https://ckan.govdata.de/ja/dataset/bebauungsplan-rahlstedt-65-hamburg']
  #testpids=['https://doi.pangaea.de/10.1594/PANGAEA.810463']
  #testpids=['https://ortus.rtu.lv/science/en/datamodule/294']
  #testpids=['https://doi.org/10.15482/USDA.ADC/1324677']
  #testpids=['https://www.proteinatlas.org/ENSG00000110651-CD81/cell']
  #testpids=['http://doi.org/10.5255/UKDA-SN-1329-1']
  #testpids=['http://purl.org/vocommons/voaf']
  #testpids=['https://meta.icos-cp.eu/objects/9ri1elaogsTv9LQFLNTfDNXm']
  #testpids=['http://doi.org/10.22033/ESGF/CMIP6.4397']
  #testpids=['https://su.figshare.com/articles/Data_for_Does_historical_land_use_affect_the_regional_distribution_of_fleshy-fruited_woody_plants_Arnell_et_al_2019_/10318046']
  #testpids=['http://doi.org/10.1007/s10531-013-0468-6']
  #rdf
  #testpids=['http://tun.fi/JX.1099769']
  testpids=['https://ortus.rtu.lv/science/en/datamodule/3']
  #rdf
  #testpids=['https://databank.ora.ox.ac.uk/UniversityCollege/datasets/04156fde-dabb-48fd-baf6-533182f74b5b']
  #testpids=['https://data.gov.lv/dati/lv/dataset/maksatnespejas-procesi']
  testpids=['http://doi.org/10.17882/42182']
  #testpids = muchotestpids
  testpids =['10.15152/QDB.121']
  startpid=''
  def effectivehandlers(logger):
      handlers = logger.handlers
      while True:
          logger = logger.parent
          handlers.extend(logger.handlers)
          if not (logger.parent and logger.propagate):
              break
      return handlers
  def main():
      config = ConfigParser.ConfigParser()
      my_path = Path(__file__).parent.parent
      ini_path = os.path.join(my_path,'config','server.ini')
      config.read(ini_path)
      YAML_DIR = config['SERVICE']['yaml_directory']
      METRIC_YAML = config['SERVICE']['metrics_yaml']
      METRIC_YML_PATH = os.path.join(my_path, YAML_DIR , METRIC_YAML)
      SPDX_URL = config['EXTERNAL']['spdx_license_github']
      DATACITE_API_REPO = config['EXTERNAL']['datacite_api_repo']
      RE3DATA_API = config['EXTERNAL']['re3data_api']
      METADATACATALOG_API = config['EXTERNAL']['metadata_catalog']
      isDebug = config.getboolean('SERVICE', 'debug_mode')
      data_files_limit = int(config['SERVICE']['data_files_limit'])
      metric_specification = config['SERVICE']['metric_specification']
  
      preproc = Preprocessor()
      preproc.retrieve_metrics_yaml(METRIC_YML_PATH, data_files_limit,metric_specification)
      print('Total metrics defined: {}'.format(preproc.get_total_metrics()))
  
      isDebug = config.getboolean('SERVICE', 'debug_mode')
      preproc.retrieve_licenses(SPDX_URL, isDebug)
      preproc.retrieve_datacite_re3repos(RE3DATA_API, DATACITE_API_REPO, isDebug)
      preproc.retrieve_metadata_standards(METADATACATALOG_API, isDebug)
      preproc.retrieve_science_file_formats(isDebug)
      preproc.retrieve_long_term_file_formats(isDebug)
  
      print('Total SPDX licenses : {}'.format(preproc.get_total_licenses()))
      print('Total re3repositories found from datacite api : {}'.format(len(preproc.getRE3repositories())))
      print('Total subjects area of imported metadata standards : {}'.format(len(preproc.metadata_standards)))
      start=False
      usedatacite = True
      tracemalloc.start()
      for identifier in testpids:
  
          print (identifier)
          if identifier==startpid or not startpid:
              start=True
          if start:
              ft = FAIRCheck(uid=identifier,  test_debug=True, use_datacite=usedatacite)
  
             # print(effectivehandlers(ft.logger))
              uid_result, pid_result = ft.check_unique_persistent()
              core_metadata_result = ft.check_minimal_metatadata()
              if ft.repeat_pid_check:
                  uid_result, pid_result = ft.check_unique_persistent()
              content_identifier_included_result = ft.check_content_identifier_included()
              access_level_result=ft.check_data_access_level()
              license_result = ft.check_license()
              relatedresources_result = ft.check_relatedresources()
              check_searchable_result = ft.check_searchable()
              data_content_metadata = ft.check_data_content_metadata()
              data_file_format_result=ft.check_data_file_format()
              community_standards_result=ft.check_community_metadatastandards()
              data_provenance_result=ft.check_data_provenance()
              formal_representation_result=ft.check_formal_metadata()
              semantic_vocabulary_result =ft.check_semantic_vocabulary()
              metadata_preserved_result = ft.check_metadata_preservation()
              standard_protocol_metadata_result = ft.check_standardised_protocol_metadata()
              standard_protocol_data_result = ft.check_standardised_protocol_data()
  
              results = [uid_result, pid_result, core_metadata_result, content_identifier_included_result, check_searchable_result, access_level_result, formal_representation_result,semantic_vocabulary_result, license_result, data_file_format_result,data_provenance_result,relatedresources_result,community_standards_result,data_content_metadata,metadata_preserved_result, standard_protocol_data_result,standard_protocol_metadata_result]
              #results=[core_metadata_result,uid_result, pid_result]
              #print(ft.metadata_merged)
              debug_messages = ft.get_log_messages_dict()
              ft.logger_message_stream.flush()
              for res_k, res_v in enumerate(results):
                  if ft.isDebug:
                      debug_list = debug_messages.get(res_v['metric_identifier'])
                      #debug_list= ft.msg_filter.getMessage(res_v['metric_identifier'])
                      if debug_list is not None:
                          results[res_k]['test_debug'] = debug_messages.get(res_v['metric_identifier'])
                      else:
                          results[res_k]['test_debug'] =['INFO: No debug messages received']
                  else:
                      results[res_k]['test_debug'] = ['INFO: Debugging disabled']
                      debug_messages = {}
              print(json.dumps(results, indent=4, sort_keys=True))
              #remove unused logger handlers and filters to avoid memory leaks
              ft.logger.handlers = [ft.logger.handlers[-1]]
              #ft.logger.filters = [ft.logger.filters]
              current, peak = tracemalloc.get_traced_memory()
              print(f"Current memory usage is {current / 10 ** 6}MB; Peak was {peak / 10 ** 6}MB")
              snapshot = tracemalloc.take_snapshot()
              top_stats = snapshot.statistics('traceback')
  
              # pick the biggest memory block
              stat = top_stats[0]
              print("%s memory blocks: %.1f KiB" % (stat.count, stat.size / 1024))
              for line in stat.traceback.format():
                  print(line)
  
              for i, stat in enumerate(snapshot.statistics('filename')[:5], 1):
                  print(i,  str(stat))
  
              #preproc.logger.
              gc.collect()
      tracemalloc.stop()
  if __name__ == '__main__':
      main()
  