import ckanapi
from harvest_helpers import *
from secret import CKAN, SOURCES

## enable one of:
#ckan = ckanapi.RemoteCKAN(CKAN["ct"]["url"], apikey=CKAN["ct"]["key"])
#ckan = ckanapi.RemoteCKAN(CKAN["ca"]["url"], apikey=CKAN["ca"]["key"])
ckan = ckanapi.RemoteCKAN(CKAN["cb"]["url"], apikey=CKAN["cb"]["key"])

print("Using CKAN {0}".format(ckan.address))

"""
Explanation: Harvesting WMS into CKAN
This notebook illustrates harvesting of a WMS endpoint into a CKAN instance.
Context
The harvested WMS endpoint belongs to Landgate's Spatial Land Information Program (SLIP). The layers within are authored by partner agencies or Landgate. There are one or several different web service endpoints per WMS layer.
Organisations
From a spreadsheet of agency references, names, and further information, CKAN organisations are initially created and subsequently used as owners of the respective harvested WMS layers.
Topics
The WMS layers are organised by topics, which will be created both as CKAN groups and keywords. Harvested datasets will be allocated to releveant CKAN groups.
Layer names
The WMS layer names contain the layer ID, consisting of agancy slug and layer reference, and the publishing date, and will be split up during harvesting.
Additional resources
Additional web service end points, as well as a list of published PDFs with further information, are added as extra resources to the CKAN datasets from harvested WMS layers.
CKAN credentials
Sensitive information and related configuration, such as CKAN URLs and credentials, are stored in a separate file.
To use this workbook on your own CKAN instance, write the following contents into a file secret.py in the same directory as this workbook:
```
CKAN = {
  "ca":{
    "url": "http://catalogue.alpha.data.wa.gov.au/",
    "key": "your-api-key" 
  },
  "cb":{
    "url": "http://catalogue.beta.data.wa.gov.au/",
    "key": "your-api-key" 
  }
}
SOURCES = {
  "NAME": {
    "proxy": "proxy_url",
    "url": "https://www2.landgate.wa.gov.au/ows/wmspublic"
  },
  ...
}
ARCGIS = {
  "SLIPFUTURE" : {
    "url": "http://services.slip.wa.gov.au/arcgis/rest/services",
    "folders": ["QC", ...]
 },
 ...
}
``
Insert your catalogue names, urls, and importantly, your write-permitted CKAN API keys.
Next we'll import the whole dictionaryCKAN`.
End of explanation
"""


wmsP = WebMapService(SOURCES["wmspublic"]["proxy"])
wmsP_url = SOURCES["wmspublic"]["url"]

wmsCM = WebMapService(SOURCES["wmsCsMosaic"]["proxy"])
wmsCM_url = SOURCES["wmsCsMosaic"]["url"]

wmsCC = WebMapService(SOURCES["wmsCsCadastre"]["proxy"])
wmsCC_url = SOURCES["wmsCsCadastre"]["url"]

wfsP = WebFeatureService(SOURCES["wfspublic_4326"]["proxy"])
wfsP_url = SOURCES["wfspublic_4326"]["url"]

wfsCA = WebFeatureService(SOURCES["wfsCsAdmin_4283"]["proxy"])
wfsCA_url = SOURCES["wfsCsAdmin_4283"]["url"]

wfsCC = WebFeatureService(SOURCES["wfsCsCadastre_4283"]["proxy"])
wfsCC_url = SOURCES["wfsCsCadastre_4283"]["url"]

#wfsCT = WebFeatureService(SOURCES["wfsCsTopo_4283"]["proxy"])
#wfsCT_url = SOURCES["wfsCsTopo_4283"]["url"]

"""
Explanation: OGC W*S endpoints
End of explanation
"""


pdfs = get_pdf_dict("data-dictionaries.csv")
org_dict = get_org_dict("organisations.csv")
group_dict = get_group_dict(wmsP)

"""
Explanation: Additional Lookups
End of explanation
"""


orgs = upsert_orgs(org_dict, ckan, debug=False)
groups = upsert_groups(group_dict, ckan, debug=False)

"""
Explanation: Create Organisations and Groups
The next step will create or update CKAN organisations from organisations.csv, and CKAN groups from WMS topics.
End of explanation
"""


l_wmsP = get_layer_dict(wmsP, wmsP_url, ckan, orgs, groups, pdfs, res_format="WMS", debug=False)
l_wmsCC = get_layer_dict(wmsCC, wmsCC_url, ckan, orgs, groups, pdfs, res_format="WMS", debug=False)
l_wmsCM = get_layer_dict(wmsCM, wmsCM_url, ckan, orgs, groups, pdfs, res_format="WMS", debug=False)
l_wfsP = get_layer_dict(wfsP, wfsP_url, ckan, orgs, groups, pdfs, res_format="WFS", debug=False)
l_wfsCA = get_layer_dict(wfsCA, wfsCA_url, ckan, orgs, groups, pdfs, res_format="WFS", debug=False)
l_wfsCC = get_layer_dict(wfsCC, wfsCC_url, ckan, orgs, groups, pdfs, res_format="WFS", debug=False)

"""
Explanation: Prepare data
The following step will prepare a dictionary of dataset metadata, ready to be inserted into CKAN. 
It parses the WMS endpoint and looks up dictionaries organisations, groups, and pdf_dict.
This step runs very quickly, as it only handles dictionaries of WMS layers, organisations and groups (both: name and id) and PDFs (name, id, url). There are no API calls to either CKAN or the WMS involved.
End of explanation
"""


# Delete all datasets with old SLIP layer id name slug
kill_list = [n for n in ckan.action.package_list() if re.match(r"(.)*-[0-9][0-9][0-9]$", n)]
#killed = [ckan.action.package_delete(id=n) for n in kill_list]
print("Killed {0} obsolete datasets".format(len(kill_list)))

"""
Explanation: Delete old datasets
Note: With great power comes great responsibility. Execute the next chunk with care and on your own risk.
End of explanation
"""


p_wmsP = upsert_datasets(l_wmsP, ckan, overwrite_metadata=True, drop_existing_resources=True)
print("{0} datasets created or updated from {1} Public WMS layers".format(len(p_wmsP), len(wmsP.contents)))

"""
Explanation: Update datasets in CKAN
First pass: add public WMS layer, overwrite metadata if dataset exists and drop any existing resources.
End of explanation
"""


p_wfs = upsert_datasets(l_wfsP, ckan, overwrite_metadata=False, drop_existing_resources=False)
print("{0} datasets created or updated from {1} public WFS layers".format(len(p_wfs), len(wfsP.contents)))

p_wmsCC = upsert_datasets(l_wmsCC, ckan, overwrite_metadata=False, drop_existing_resources=False, debug=False)
print("{0} datasets created or updated from {1} Cadastre WMS layers".format(len(p_wmsCC), len(wmsCC.contents)))

p_wfsCC = upsert_datasets(l_wfsCC, ckan, overwrite_metadata=False, drop_existing_resources=False)
print("{0} datasets created or updated from {1} Cadastre WFS layers".format(len(p_wfsCC), len(wfsCC.contents)))

p_wfsCA = upsert_datasets(l_wfsCA, ckan, overwrite_metadata=False, drop_existing_resources=False)
print("{0} datasets created or updated from {1} Cadastre Admin WFS layers".format(len(p_wfsCA), len(wfsCA.contents)))

"""
Explanation: Second pass: add public WFS, but retain metadata and resources of existing datasets. Repeat this mode for remaining sources.
End of explanation
"""


