accessmod/osm.py
def extract_from_osm(config: str, webhook_url: str, webhook_token: str):
    """Download and extract transport network and water data from OpenStreetMap."""
    logger.info("extract_from_osm() make work dir")

    # config is a json file
    if config.endswith(".json"):
        fs = utils.filesystem(config)
        with fs.open(config) as f:
            config = json.load(f)
    # config is base64 encoded json string
    else:
        config = json.loads(base64.b64decode(config))

    os.makedirs(WORK_DIR, exist_ok=True)

    target_geometry = utils.parse_extent(config["extent"])
    all_countries = gpd.read_file("countries_pbf.json")
    all_countries["localpath"] = ""
    all_countries["localpbf"] = all_countries["pbf"].apply(
        lambda p: p.replace("/", "-")
    )

    logger.info("extract_from_osm() download source data")
    countries = all_countries[all_countries.intersects(target_geometry)]
    for i, country in countries.iterrows():
        localpath = os.path.join(WORK_DIR, country["localpbf"])
        countries.loc[i, "localpath"] = localpath
        r = requests.get("http://download.geofabrik.de/" + country["pbf"])
        r.raise_for_status()
        open(localpath, "wb").write(r.content)
        logger.info("Downloaded %s to %s", country["pbf"], localpath)

    if config["transport_network"]["auto"]:
        logger.info("extract_from_osm() transport_network")
        transport_file = os.path.join(WORK_DIR, "transport.gpkg")
        extract_pbf(
            list(countries.localpath),
            transport_file,
            target_geometry,
            ["w/highway", "w/route=ferry"],
            [
                "highway",
                "smoothness",
                "surface",
                "tracktype",
                "route",
                "duration",
                "motor_vehicle",
                "motorcar",
                "motorcycle",
                "bicycle",
                "foot",
            ],
        )
        utils.upload_file(
            transport_file, config["transport_network"]["path"], config["overwrite"]
        )

        # save a geojson copy of the dataset for dataviz purposes
        with tempfile.NamedTemporaryFile(suffix=".geojson") as tmp_file:
            geojson_uri = utils.fpath_suffix(
                src_fpath=config["transport_network"]["path"],
                suffix="web",
                dst_extension="geojson",
            )
            geojson_tmp = processing.generate_geojson(transport_file, tmp_file.name)
            utils.upload_file(geojson_tmp, geojson_uri, config.get("overwrite", True))

        # columns and unique values
        metadata = {
            "columns": ["highway", "smoothness", "surface", "tracktype"],
            "values": {
                "highway": []  # todo: for now these values are hardcoded in the front-end
            },
            "category_column": "highway",
            "geojson_uri": geojson_uri,
        }

        utils.call_webhook(
            event_type="acquisition_completed",
            data={
                "acquisition_type": "transport_network",
                "uri": config["transport_network"]["path"],
                "mime_type": "application/geopackage+sqlite3",
                "metadata": metadata,
            },
            url=webhook_url,
            token=webhook_token,
        )

    if config["water"]["auto"]:
        logger.info("extract_from_osm() water")
        water_file = os.path.join(WORK_DIR, "water.gpkg")
        extract_pbf(
            list(countries.localpath),
            water_file,
            target_geometry,
            ["nwr/natural=water", "nwr/waterway", "nwr/water"],
            ["waterway", "natural", "water", "wetland", "boat"],
        )
        utils.upload_file(water_file, config["water"]["path"], config["overwrite"])
        utils.call_webhook(
            event_type="acquisition_completed",
            data={
                "acquisition_type": "water",
                "uri": config["water"]["path"],
                "mime_type": "application/geopackage+sqlite3",
            },
            url=webhook_url,
            token=webhook_token,
        )
    logger.info("extract_from_osm() finished")
Download and extract transport network and water data from OpenStreetMap.
Download and extract transport network and water data from OpenStreetMap.
Download and extract transport network and water data from OpenStreetMap .
def extract_from_osm(config: str, webhook_url: str, webhook_token: str):
    logger.info("extract_from_osm() make work dir")
    if config.endswith(".json"):
        fs = utils.filesystem(config)
        with fs.open(config) as f:
            config = json.load(f)
    else:
        config = json.loads(base64.b64decode(config))
    os.makedirs(WORK_DIR, exist_ok=True)
    target_geometry = utils.parse_extent(config["extent"])
    all_countries = gpd.read_file("countries_pbf.json")
    all_countries["localpath"] = ""
    all_countries["localpbf"] = all_countries["pbf"].apply(
        lambda p: p.replace("/", "-")
    )
    logger.info("extract_from_osm() download source data")
    countries = all_countries[all_countries.intersects(target_geometry)]
    for i, country in countries.iterrows():
        localpath = os.path.join(WORK_DIR, country["localpbf"])
        countries.loc[i, "localpath"] = localpath
        r = requests.get("http://download.geofabrik.de/" + country["pbf"])
        r.raise_for_status()
        open(localpath, "wb").write(r.content)
        logger.info("Downloaded %s to %s", country["pbf"], localpath)
    if config["transport_network"]["auto"]:
        logger.info("extract_from_osm() transport_network")
        transport_file = os.path.join(WORK_DIR, "transport.gpkg")
        extract_pbf(
            list(countries.localpath),
            transport_file,
            target_geometry,
            ["w/highway", "w/route=ferry"],
            [
                "highway",
                "smoothness",
                "surface",
                "tracktype",
                "route",
                "duration",
                "motor_vehicle",
                "motorcar",
                "motorcycle",
                "bicycle",
                "foot",
            ],
        )
        utils.upload_file(
            transport_file, config["transport_network"]["path"], config["overwrite"]
        )
        with tempfile.NamedTemporaryFile(suffix=".geojson") as tmp_file:
            geojson_uri = utils.fpath_suffix(
                src_fpath=config["transport_network"]["path"],
                suffix="web",
                dst_extension="geojson",
            )
            geojson_tmp = processing.generate_geojson(transport_file, tmp_file.name)
            utils.upload_file(geojson_tmp, geojson_uri, config.get("overwrite", True))
        metadata = {
            "columns": ["highway", "smoothness", "surface", "tracktype"],
            "values": {
                "highway": []  
            },
            "category_column": "highway",
            "geojson_uri": geojson_uri,
        }
        utils.call_webhook(
            event_type="acquisition_completed",
            data={
                "acquisition_type": "transport_network",
                "uri": config["transport_network"]["path"],
                "mime_type": "application/geopackage+sqlite3",
                "metadata": metadata,
            },
            url=webhook_url,
            token=webhook_token,
        )
    if config["water"]["auto"]:
        logger.info("extract_from_osm() water")
        water_file = os.path.join(WORK_DIR, "water.gpkg")
        extract_pbf(
            list(countries.localpath),
            water_file,
            target_geometry,
            ["nwr/natural=water", "nwr/waterway", "nwr/water"],
            ["waterway", "natural", "water", "wetland", "boat"],
        )
        utils.upload_file(water_file, config["water"]["path"], config["overwrite"])
        utils.call_webhook(
            event_type="acquisition_completed",
            data={
                "acquisition_type": "water",
                "uri": config["water"]["path"],
                "mime_type": "application/geopackage+sqlite3",
            },
            url=webhook_url,
            token=webhook_token,
        )
    logger.info("extract_from_osm() finished")
def extract_from_osm ( config : str , webhook_url : str , webhook_token : str ) : logger . info ( "extract_from_osm() make work dir" ) if config . endswith ( ".json" ) : fs = utils . filesystem ( config ) with fs . open ( config ) as f : config = json . load ( f ) else : config = json . loads ( base64 . b64decode ( config ) ) os . makedirs ( WORK_DIR , exist_ok = True ) target_geometry = utils . parse_extent ( config [ "extent" ] ) all_countries = gpd . read_file ( "countries_pbf.json" ) all_countries [ "localpath" ] = "" all_countries [ "localpbf" ] = all_countries [ "pbf" ] . apply ( lambda p : p . replace ( "/" , "-" ) ) logger . info ( "extract_from_osm() download source data" ) countries = all_countries [ all_countries . intersects ( target_geometry ) ] for i , country in countries . iterrows ( ) : localpath = os . path . join ( WORK_DIR , country [ "localpbf" ] ) countries . loc [ i , "localpath" ] = localpath r = requests . get ( "http://download.geofabrik.de/" + country [ "pbf" ] ) r . raise_for_status ( ) open ( localpath , "wb" ) . write ( r . content ) logger . info ( "Downloaded %s to %s" , country [ "pbf" ] , localpath ) if config [ "transport_network" ] [ "auto" ] : logger . info ( "extract_from_osm() transport_network" ) transport_file = os . path . join ( WORK_DIR , "transport.gpkg" ) extract_pbf ( list ( countries . localpath ) , transport_file , target_geometry , [ "w/highway" , "w/route=ferry" ] , [ "highway" , "smoothness" , "surface" , "tracktype" , "route" , "duration" , "motor_vehicle" , "motorcar" , "motorcycle" , "bicycle" , "foot" , ] , ) utils . upload_file ( transport_file , config [ "transport_network" ] [ "path" ] , config [ "overwrite" ] ) with tempfile . NamedTemporaryFile ( suffix = ".geojson" ) as tmp_file : geojson_uri = utils . fpath_suffix ( src_fpath = config [ "transport_network" ] [ "path" ] , suffix = "web" , dst_extension = "geojson" , ) geojson_tmp = processing . generate_geojson ( transport_file , tmp_file . name ) utils . upload_file ( geojson_tmp , geojson_uri , config . get ( "overwrite" , True ) ) metadata = { "columns" : [ "highway" , "smoothness" , "surface" , "tracktype" ] , "values" : { "highway" : [ ] } , "category_column" : "highway" , "geojson_uri" : geojson_uri , } utils . call_webhook ( event_type = "acquisition_completed" , data = { "acquisition_type" : "transport_network" , "uri" : config [ "transport_network" ] [ "path" ] , "mime_type" : "application/geopackage+sqlite3" , "metadata" : metadata , } , url = webhook_url , token = webhook_token , ) if config [ "water" ] [ "auto" ] : logger . info ( "extract_from_osm() water" ) water_file = os . path . join ( WORK_DIR , "water.gpkg" ) extract_pbf ( list ( countries . localpath ) , water_file , target_geometry , [ "nwr/natural=water" , "nwr/waterway" , "nwr/water" ] , [ "waterway" , "natural" , "water" , "wetland" , "boat" ] , ) utils . upload_file ( water_file , config [ "water" ] [ "path" ] , config [ "overwrite" ] ) utils . call_webhook ( event_type = "acquisition_completed" , data = { "acquisition_type" : "water" , "uri" : config [ "water" ] [ "path" ] , "mime_type" : "application/geopackage+sqlite3" , } , url = webhook_url , token = webhook_token , ) logger . info ( "extract_from_osm() finished" )
Download and extract transport network and water data from OpenStreetMap.
Download and extract transport network and water data from OpenStreetMap .
"""Download and extract transport network and water data from OpenStreetMap.""" # config is a json file # config is base64 encoded json string # save a geojson copy of the dataset for dataviz purposes # columns and unique values # todo: for now these values are hardcoded in the front-end
config
str
webhook_url
str
webhook_token
str