filling missing values, include all gap codes

preprocess.ipynb

@@ -12,114 +12,80 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "f7e6298c-d886-432a-a1b7-c3fee914c24f",
-   "metadata": {},
+   "metadata": {
+    "editable": true,
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "import ibis\n",
     "from ibis import _\n",
     "\n",
-    "conn = ibis.duckdb.connect(\"tmp\", extensions=[\"spatial\"])\n",
-    "ca_parquet = \"https://data.source.coop/cboettig/ca30x30/ca_areas.parquet\"\n",
+    "conn = ibis.duckdb.connect(\"tmp3\", extensions=[\"spatial\"])\n",
+    "# ca_parquet = \"https://data.source.coop/cboettig/ca30x30/ca_areas.parquet\"\n",
     "# or use local copy:\n",
-    "ca_parquet = \"/home/rstudio/source.coop/cboettig/ca30x30/ca_areas.parquet\"\n"
+    "ca_parquet = \"ca_areas.parquet\""
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "a0cb34b1-8d70-49bf-80c6-244ecc8ddf84",
+   "id": "a3d4f189-1563-4868-9f1f-64d67569df27",
    "metadata": {},
    "outputs": [],
    "source": [
-    "buffer = -2\n",
+    "# negative buffer to account for overlapping boundaries. \n",
+    "buffer = -30 #30m buffer \n",
     "\n",
     "tbl = (\n",
     "    conn.read_parquet(ca_parquet)\n",
     "    .cast({\"SHAPE\": \"geometry\"})\n",
     "    .rename(geom = \"SHAPE\")\n",
-    "#    .filter(_.UNIT_NAME == \"Angeles National Forest\")\n",
-    "    .filter(_.reGAP < 3) \n",
+    "    .filter(_.reGAP < 3) # only gap 1 and 2 count towards 30x30\n",
     ")\n",
-    "tbl_2023 = tbl.filter(_.Release_Year == 2023).mutate(geom=_.geom.buffer(buffer))\n",
+    "\n",
+    "# polygons with release_year 2024 are a superset of release_year 2023. \n",
+    "# use anti_join to isolate the objects that are in release_year 2024 but not release_year 2023 (aka newly established). \n",
+    "tbl_2023 = tbl.filter(_.Release_Year == 2023).mutate(geom=_.geom.buffer(buffer)) \n",
     "tbl_2024 = tbl.filter(_.Release_Year == 2024)\n",
-    "intersects = tbl_2024.anti_join(tbl_2023, _.geom.intersects(tbl_2023.geom))\n",
-    "\n"
+    "intersects = tbl_2024.anti_join(tbl_2023, _.geom.intersects(tbl_2023.geom))"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "a0b75637-e015-4be4-86e1-c9757ac43d0f",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "## Testing, run only on subset data\n",
-    "if False:\n",
-    "    gdf = intersects.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute()\n",
-    "    gdf_2023 = tbl_2023.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute()\n",
-    "    gdf_2024 = tbl_2024.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute()\n",
-    "    # gdf = ca2024\n",
-    "    established = {'property': 'established',\n",
-    "                   'type': 'categorical',\n",
-    "                   'stops': [\n",
-    "                       [2023, \"#26542C80\"], \n",
-    "                       [2024, \"#F3AB3D80\"]]\n",
-    "                  }\n",
-    "    inter = {\"fill-color\": \"#F3AB3D\"}\n",
-    "    p2024 = {\"fill-color\": \"#26542C\"}\n",
-    "    p2023 = {\"fill-color\": \"#8B0A1A\"}\n",
-    "    \n",
-    "    m = leafmap.Map(style=\"positron\")\n",
-    "    m.add_gdf(gdf_2024,layer_type=\"fill\", name = \"2024\", paint = p2024)\n",
-    "    m.add_gdf(gdf_2023,layer_type=\"fill\", name = \"2023\", paint = p2023)\n",
-    "    m.add_gdf(gdf,layer_type=\"fill\", name = \"intersects\", paint = inter)\n",
-    "    \n",
-    "    m.add_layer_control()\n",
-    "    m"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "275c171a-f82f-4ee8-991c-1e34eb83a33d",
+   "id": "a59c976b-3c36-40f9-a15b-cefcd155c647",
    "metadata": {},
    "outputs": [],
    "source": [
     "%%time\n",
     "\n",
-    "new2024 = intersects.select(\"OBJECTID\").mutate(established = 2024)\n",
+    "new2024 = intersects.select(\"OBJECTID\").mutate(established = 2024) # saving IDs to join on\n",
     "\n",
     "ca = (conn\n",
     "      .read_parquet(ca_parquet)\n",
     "      .cast({\"SHAPE\": \"geometry\"})\n",
     "      .mutate(area = _.SHAPE.area())\n",
-    "      .filter(_.Release_Year == 2024)\n",
-    "      .filter(_.reGAP < 3)\n",
-    "      .left_join(new2024, \"OBJECTID\")\n",
-    "      .mutate(established=_.established.fill_null(2023))\n",
+    "      .filter(_.Release_Year == 2024) # having both 2023 and 2024 is redudant since 2024 is the superset.\n",
+    "      .left_join(new2024, \"OBJECTID\") # newly established 2024 polygons \n",
+    "      .mutate(established=_.established.fill_null(2023)) \n",
     "      .mutate(geom = _.SHAPE.convert(\"epsg:3310\",\"epsg:4326\"))\n",
     "      .rename(name = \"cpad_PARK_NAME\", access_type = \"cpad_ACCESS_TYP\", manager = \"cpad_MNG_AGENCY\",\n",
     "              manager_type = \"cpad_MNG_AG_LEV\", id = \"OBJECTID\", type = \"TYPE\")\n",
+    "      .mutate(manager = _.manager.substitute({\"\": \"Unknown\"})) \n",
+    "      .mutate(manager_type = _.manager_type.substitute({\"\": \"Unknown\"}))\n",
+    "      .mutate(access_type = _.access_type.substitute({\"\": \"Unknown Access\"}))\n",
+    "      .mutate(name = _.name.substitute({\"\": \"Unknown\"}))\n",
     "      .select(_.established, _.reGAP, _.name, _.access_type, _.manager, _.manager_type,\n",
     "              _.Easement, _.Acres, _.id, _.type, _.geom)\n",
     "     )\n",
-    "ca2024 = ca.execute()\n",
-    "\n",
     "\n",
-    "\n",
-    "ca2024.to_parquet(\"ca2024.parquet\")\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "8259b450-2152-472c-a58c-50ce0d68d78f",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "ca2024 = conn.read_parquet(\"ca2024.parquet\")\n",
-    "ca2024.execute().to_file(\"ca2024.geojson\") # tippecanoe can't parse geoparquet :-("
+    "ca2024 = ca.execute()\n",
+    "ca2024.to_parquet(\"ca2024-30m.parquet\")\n",
+    "ca2024.to_file(\"ca2024-30m.geojson\") # tippecanoe can't parse geoparquet :-("
    ]
   },
   {
@@ -144,83 +110,21 @@
     "            repo_id=\"boettiger-lab/ca-30x30\",\n",
     "            repo_type=\"dataset\",\n",
     "        )\n",
-    "hf_upload(\"ca2024.parquet\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "cebd0ff5-8353-4b84-b9ee-182b74613554",
-   "metadata": {},
-   "source": [
-    "# Testing & visualization\n",
-    "\n",
-    "`ca2024.parquet()` now contains all we need.  The code below illustrates some quick examples of the kinds of visualizations and summaries we might want to compute with this data.  \n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "55afe07c-8681-4308-bbb9-e460f7380f86",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import leafmap.maplibregl as leafmap\n",
-    "import ibis\n",
-    "from ibis import _\n",
-    "conn = ibis.duckdb.connect(extensions=[\"spatial\"])\n",
-    "\n",
-    "ca2024 = conn.read_parquet(\"ca2024.parquet\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "6f3df8c1-a603-4dd5-be84-8deaae928d0a",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# compute some summary tables:\n",
-    "\n",
-    "(ca2024\n",
-    " .filter(_.established == 2024)\n",
-    " .filter(_.manager_type == \"State\")\n",
-    " .group_by(_.manager, _.manager_type)\n",
-    " .agg(area = _.Acres.sum())\n",
-    " .order_by(_.area.desc())\n",
-    " .execute()\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c62854f6-1456-4207-8c69-53af17970102",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "\n",
-    "gdf = ca2024.execute()\n",
-    "established = {'property': 'established',\n",
-    "               'type': 'categorical',\n",
-    "               'stops': [\n",
-    "                   [2023, \"#26542C80\"], \n",
-    "                   [2024, \"#F3AB3D80\"]]}\n",
-    "paint = {\"fill-color\": established}\n",
-    "\n",
-    "\n",
-    "m = leafmap.Map(style=\"positron\")\n",
-    "m.add_gdf(gdf,layer_type=\"fill\", name = \"intersects\", paint = paint)\n",
-    "\n",
-    "m.add_layer_control()\n",
-    "m.to_html(\"ca2024.html\")\n",
-    "m"
+    "    \n",
+    "hf_upload(\"ca2024-30m.parquet\")"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
    "id": "2df80e1d-6b94-4884-b9f5-d9c23d3ea028",
-   "metadata": {},
+   "metadata": {
+    "editable": true,
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "import subprocess\n",
@@ -249,8 +153,8 @@
     "    except subprocess.CalledProcessError as e:\n",
     "        print(f\"Error running Tippecanoe: {e}\")\n",
     "\n",
-    "generate_pmtiles(\"ca2024.geojson\", \"ca2024-tippe.pmtiles\")\n",
-    "hf_upload(\"ca2024-tippe.pmtiles\")"
+    "generate_pmtiles(\"ca2024-30m.geojson\", \"ca2024-30m-tippe.pmtiles\")\n",
+    "hf_upload(\"ca2024-30m-tippe.pmtiles\")"
    ]
   }
  ],

preprocess.py

@@ -0,0 +1,99 @@
+import ibis
+from ibis import _
+
+conn = ibis.duckdb.connect("tmp3", extensions=["spatial"])
+# ca_parquet = "https://data.source.coop/cboettig/ca30x30/ca_areas.parquet"
+# or use local copy:
+ca_parquet = "ca_areas.parquet"
+
+
+# negative buffer to account for overlapping boundaries. 
+buffer = -30 #30m buffer 
+
+tbl = (
+    conn.read_parquet(ca_parquet)
+    .cast({"SHAPE": "geometry"})
+    .rename(geom = "SHAPE")
+    .filter(_.reGAP < 3) # only gap 1 and 2 count towards 30x30
+)
+
+# polygons with release_year 2024 are a superset of release_year 2023. 
+# use anti_join to isolate the objects that are in release_year 2024 but not release_year 2023 (aka newly established). 
+tbl_2023 = tbl.filter(_.Release_Year == 2023).mutate(geom=_.geom.buffer(buffer)) 
+tbl_2024 = tbl.filter(_.Release_Year == 2024)
+intersects = tbl_2024.anti_join(tbl_2023, _.geom.intersects(tbl_2023.geom))
+
+new2024 = intersects.select("OBJECTID").mutate(established = 2024) # saving IDs to join on
+
+ca = (conn
+      .read_parquet(ca_parquet)
+      .cast({"SHAPE": "geometry"})
+      .mutate(area = _.SHAPE.area())
+      .filter(_.Release_Year == 2024) # having both 2023 and 2024 is redudant since 2024 is the superset.
+      .left_join(new2024, "OBJECTID") # newly established 2024 polygons 
+      .mutate(established=_.established.fill_null(2023)) 
+      .mutate(geom = _.SHAPE.convert("epsg:3310","epsg:4326"))
+      .rename(name = "cpad_PARK_NAME", access_type = "cpad_ACCESS_TYP", manager = "cpad_MNG_AGENCY",
+              manager_type = "cpad_MNG_AG_LEV", id = "OBJECTID", type = "TYPE")
+      .mutate(manager = _.manager.substitute({"": "Unknown"})) 
+      .mutate(manager_type = _.manager_type.substitute({"": "Unknown"}))
+      .mutate(access_type = _.access_type.substitute({"": "Unknown Access"}))
+      .mutate(name = _.name.substitute({"": "Unknown"}))
+      .select(_.established, _.reGAP, _.name, _.access_type, _.manager, _.manager_type,
+              _.Easement, _.Acres, _.id, _.type, _.geom)
+     )
+
+ca2024 = ca.execute()
+
+ca2024.to_parquet("ca2024-30m.parquet")
+
+ca2024.to_file("ca2024-30m.geojson") # tippecanoe can't parse geoparquet :-(
+
+
+## Upload to Huggingface
+# https://huggingface.co/datasets/boettiger-lab/ca-30x30/
+
+from huggingface_hub import HfApi, login
+import streamlit as st
+login(st.secrets["HF_TOKEN"])
+api = HfApi()
+
+def hf_upload(file):
+    info = api.upload_file(
+            path_or_fileobj=file,
+            path_in_repo=file,
+            repo_id="boettiger-lab/ca-30x30",
+            repo_type="dataset",
+        )
+hf_upload("ca2024-30m.parquet")
+
+
+
+import subprocess
+import os
+
+def generate_pmtiles(input_file, output_file, max_zoom=12):
+    # Ensure Tippecanoe is installed
+    if subprocess.call(["which", "tippecanoe"], stdout=subprocess.DEVNULL) != 0:
+        raise RuntimeError("Tippecanoe is not installed or not in PATH")
+
+    # Construct the Tippecanoe command
+    command = [
+        "tippecanoe",
+        "-o", output_file,
+        "-z", str(max_zoom),
+        "--drop-densest-as-needed",
+        "--extend-zooms-if-still-dropping",
+        "--force",
+        input_file
+    ]
+
+    # Run Tippecanoe
+    try:
+        subprocess.run(command, check=True)
+        print(f"Successfully generated PMTiles file: {output_file}")
+    except subprocess.CalledProcessError as e:
+        print(f"Error running Tippecanoe: {e}")
+
+generate_pmtiles("ca2024-30m.geojson", "ca2024-30m-tippe.pmtiles")
+hf_upload("ca2024-30m-tippe.pmtiles")