Merge pull request #21 from connorsutton/main

.github/workflows/docker-hub.yml

@@ -23,6 +23,7 @@ jobs:
     - name: Build Docker image
       run: |
         docker build \
+          --target runtime \
           -t joshuasundance/geospatial-data-converter:${{ github.ref_name }} \
           -t joshuasundance/geospatial-data-converter:latest \
           .

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11-slim-bookworm
+FROM python:3.11-slim-bookworm AS base
 
 RUN adduser --uid 1000 --disabled-password --gecos '' appuser
 USER 1000
@@ -10,11 +10,18 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
 RUN pip install --user --no-cache-dir --upgrade pip
 COPY ./requirements.txt /home/appuser/requirements.txt
 RUN pip install --user --no-cache-dir  --upgrade -r /home/appuser/requirements.txt
-
 COPY geospatial-data-converter/ /home/appuser/geospatial-data-converter/
 
+
+FROM base AS test
+
+COPY dev-requirements.txt /home/appuser/dev-requirements.txt
+RUN pip install --user --no-cache-dir --upgrade -r /home/appuser/dev-requirements.txt
+
+
+FROM base AS runtime
+
 WORKDIR /workspace
 EXPOSE 7860
 
 CMD ["streamlit", "run", "/home/appuser/geospatial-data-converter/app.py", "--server.port", "7860", "--server.address", "0.0.0.0", "--server.enableXsrfProtection=false"]
-#CMD ["/bin/bash"]

README.md

@@ -39,7 +39,7 @@ This project showcases a simple geospatial data converter using [Streamlit](http
 - User-friendly interface for easy data conversion
 - Supports conversion from the following input formats:
   - ArcGIS featurelayer URL
-  - Uploaded file: KML, KMZ, GeoJSON, ZIP
+  - Uploaded file: KML, KMZ, GeoJSON, ZIP, etc
 - Provides data in the selected output format
 - Presents data preview (geometry omitted for display purposes)
 - Download button for the converted data
@@ -57,6 +57,9 @@ This project showcases a simple geospatial data converter using [Streamlit](http
 2. Run in terminal: `docker compose up`
 3. Open http://localhost:7860 in your browser
 
+## Run Tests (with local Docker container)
+1. Run in terminal: `docker compose run test`
+
 ## Kubernetes
 1. Clone the repo. Navigate to cloned repo directory
 2. Run bash script: `/bin/bash ./kubernetes/deploy.sh`

dev-requirements.txt

@@ -0,0 +1,2 @@
+pytest==7.4.3
+pytest-cov==4.1.0

docker-compose.yml

@@ -1,16 +1,26 @@
 version: '3.8'
 
 services:
+  test:
+    container_name: geospatial-data-converter-test
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: test
+    volumes:
+      - ./tests/:/home/appuser/tests/
+      - ./pyproject.toml:/home/appuser/pyproject.toml
+    working_dir: /home/appuser
+    command: ["pytest"]
+
   geospatial-data-converter:
-    image: geospatial-data-converter:latest
-    build: .
+    container_name: geospatial-data-converter
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: runtime
+    depends_on:
+      test:
+        condition: service_completed_successfully
     ports:
       - "${APP_PORT:-7860}:${APP_PORT:-7860}"
-    working_dir: /workspace
-    command: [
-      "streamlit", "run",
-      "/home/appuser/geospatial-data-converter/app.py",
-      "--server.port", "${APP_PORT:-7860}",
-      "--server.address", "0.0.0.0",
-      "--server.enableXsrfProtection=false"
-    ]

geospatial-data-converter/app.py

@@ -68,8 +68,6 @@ elif st.session_state.uploaded_file is not None:
         os.path.basename(st.session_state.uploaded_file.name),
     )
     st.session_state.gdf = read_file(st.session_state.uploaded_file)
-    # except AttributeError:
-    #     pass # there is a lingering file from the previous instance, but it can be ignored
 
 if st.session_state.gdf is not None:
     st.selectbox(

geospatial-data-converter/kml_tricks.py

@@ -52,7 +52,7 @@ def parse_descriptions_to_geodf(geodf: gpd.GeoDataFrame) -> gpd.GeoDataFrame:
 
 def swap_coordinates(geometry):
     """
-    Swap the latitude and longitude of Shapely Point, LineString, Polygon,
+    Swap the latitude and longitude of a shapely Point, LineString, Polygon,
     MultiPoint, MultiLineString, MultiPolygon, or LinearRing geometry.
 
     Parameters:
@@ -131,7 +131,7 @@ def extract_data_from_kml_code(kml_code: str) -> pd.DataFrame:
     """Extracts data from KML code into a DataFrame using SimpleData tags, excluding embedded tables in feature descriptions"""
 
     # Parse the KML source code
-    soup = bs4.BeautifulSoup(kml_code, "html.parser")
+    soup = bs4.BeautifulSoup(kml_code, features="xml")
 
     # Find all SchemaData tags (representing rows)
     schema_data_tags = soup.find_all("schemadata")

geospatial-data-converter/utils.py

@@ -1,18 +1,20 @@
 import io
 import os
 import zipfile
-from tempfile import TemporaryDirectory
-from typing import BinaryIO
 import geopandas as gpd
+import topojson
 
+from tempfile import TemporaryDirectory
+from typing import BinaryIO
 from kml_tricks import load_ge_data
 
 output_format_dict = {
-    "ESRI Shapefile": ("shp", "zip", "application/zip"),  # must be zipped
-    "OpenFileGDB": ("gdb", "zip", "application/zip"),  # must be zipped
-    "GeoJSON": ("geojson", "geojson", "application/geo+json"),
     "CSV": ("csv", "csv", "text/csv"),
     "KML": ("kml", "kml", "application/vnd.google-earth.kml+xml"),
+    "GeoJSON": ("geojson", "geojson", "application/geo+json"),
+    "TopoJSON": ("topojson", "topojson", "application/json"),
+    "ESRI Shapefile": ("shp", "zip", "application/zip"),  # must be zipped
+    "OpenFileGDB": ("gdb", "zip", "application/zip"),  # must be zipped
 }
 
 
@@ -22,7 +24,7 @@ def read_file(file: BinaryIO, *args, **kwargs) -> gpd.GeoDataFrame:
     ext = ext.lower().strip(".")
     if ext == "zip":
         with TemporaryDirectory() as tmp_dir:
-            tmp_file_path = os.path.join(tmp_dir, file.name)
+            tmp_file_path = os.path.join(tmp_dir, f"{basename}.{ext}")
             with open(tmp_file_path, "wb") as tmp_file:
                 tmp_file.write(file.read())
             return gpd.read_file(
@@ -33,7 +35,7 @@ def read_file(file: BinaryIO, *args, **kwargs) -> gpd.GeoDataFrame:
             )
     elif ext in ("kml", "kmz"):
         with TemporaryDirectory() as tmp_dir:
-            tmp_file_path = os.path.join(tmp_dir, file.name)
+            tmp_file_path = os.path.join(tmp_dir, f"{basename}.{ext}")
             with open(tmp_file_path, "wb") as tmp_file:
                 tmp_file.write(file.read())
             return load_ge_data(tmp_file_path)
@@ -60,13 +62,12 @@ def convert(gdf: gpd.GeoDataFrame, output_name: str, output_format: str) -> byte
     """Convert a GeoDataFrame to the specified format"""
     with TemporaryDirectory() as tmpdir:
         out_path = os.path.join(tmpdir, output_name)
-
         if output_format == "CSV":
             gdf.to_csv(out_path)
+        elif output_format == "TopoJSON":
+            topojson_data = topojson.Topology(gdf)
+            topojson_data.to_json(out_path)
         else:
-            # if output_format == 'KML': # need to longitude and latitude columns
-            #     # gdf = gdf.apply(lambda row: swap_coordinates(row), axis=1)
-            #     gdf['geometry'] = gdf['geometry'].apply(swap_coordinates)
             gdf.to_file(out_path, driver=output_format, engine="pyogrio")
 
         if output_format in ("ESRI Shapefile", "OpenFileGDB"):

bumpver.toml → pyproject.toml

@@ -1,4 +1,9 @@
-[bumpver]
+[tool.pytest.ini_options]
+pythonpath = ["geospatial-data-converter"]
+testpaths = ["tests"]
+addopts = "--cov=geospatial-data-converter"
+
+[tool.bumpver]
 current_version = "0.2.1"
 version_pattern = "MAJOR.MINOR.PATCH"
 commit_message = "bump version {old_version} -> {new_version}"
@@ -10,7 +15,7 @@ commit = true
 tag = true
 push = true
 
-[bumpver.file_patterns]
+[tool.bumpver.file_patterns]
 "bumpver.toml" = [
     'current_version = "{version}"',
 ]

requirements.txt

@@ -2,6 +2,8 @@ aiohttp==3.9.1
 beautifulsoup4==4.12.2
 geopandas==0.14.1
 lxml==4.9.3
+numpy==1.26.2  # pinned for snyk
 pyogrio==0.7.2
 restgdf==0.9.7
 streamlit==1.29.0
+topojson==1.7

tests/test_conversions.py

@@ -0,0 +1,19 @@
+import os
+import pytest
+from utils import convert, read_file, output_format_dict
+
+input_exts = ["kml", "kmz", "geojson", "zip"]
+output_exts = output_format_dict.keys()
+
+
+@pytest.mark.parametrize("in_ext", input_exts)
+@pytest.mark.parametrize("out_ext", output_exts)
+def test_coversion(in_ext: str, out_ext: str) -> None:
+    test_file = f"test.{in_ext}"
+    test_file_path = os.path.join(os.getcwd(), "tests", "test_data", test_file)
+    with open(test_file_path, "rb") as f:
+        in_file = read_file(f)
+    out_file = f"test.{output_format_dict[out_ext][0]}"
+    converted_data = convert(in_file, out_file, out_ext)
+    with open("test.kml", "wb") as f:
+        f.write(converted_data)