diff --git a/app/models/pydantic/creation_options.py b/app/models/pydantic/creation_options.py index 4a7260399..02a4a1f9d 100644 --- a/app/models/pydantic/creation_options.py +++ b/app/models/pydantic/creation_options.py @@ -399,6 +399,11 @@ class StaticVectorTileCacheCreationOptions(TileCacheBaseModel): "for vector tile caches. `source` and `source-layer` attributes must use `dataset` name." "Styling rules will be used in autogenerated root.json and preview.", ) + feature_filter: Optional[Dict[str, Any]] = Field( + None, + description="Optional tippecanoe feature filter(s). Uses the syntax of " + "[Mapbox legacy filters](https://docs.mapbox.com/style-spec/reference/other/#other-filters)" + ) class StaticVectorFileCreationOptions(StrictBaseModel): diff --git a/app/tasks/raster_tile_cache_assets/raster_tile_cache_assets.py b/app/tasks/raster_tile_cache_assets/raster_tile_cache_assets.py index 93bf63e0e..2d4bd078c 100644 --- a/app/tasks/raster_tile_cache_assets/raster_tile_cache_assets.py +++ b/app/tasks/raster_tile_cache_assets/raster_tile_cache_assets.py @@ -32,6 +32,7 @@ from ...errors import RecordNotFoundError + async def raster_tile_cache_asset( dataset: str, version: str, diff --git a/app/tasks/static_vector_tile_cache_assets.py b/app/tasks/static_vector_tile_cache_assets.py index 42e82f10c..306093e9c 100644 --- a/app/tasks/static_vector_tile_cache_assets.py +++ b/app/tasks/static_vector_tile_cache_assets.py @@ -3,6 +3,8 @@ from typing import Any, Dict, List, Optional from uuid import UUID +from fastapi.encoders import jsonable_encoder + from ..crud import assets, metadata from ..errors import RecordNotFoundError from ..models.orm.assets import Asset as ORMAsset @@ -55,7 +57,9 @@ async def static_vector_tile_cache_asset( # Create NDJSON asset as side effect ############################ - ndjson_uri = get_asset_uri(dataset, version, AssetType.ndjson) + ndjson_uri = get_asset_uri( + dataset, version, AssetType.ndjson, creation_options.dict() + ) ndjson_asset: ORMAsset = await assets.create_asset( dataset, @@ -77,7 +81,7 @@ async def static_vector_tile_cache_asset( "-v", version, "-f", - f"{dataset}_{version}.ndjson", + f"{creation_options.implementation}.ndjson", "-F", "GeoJSONSeq", "-T", @@ -112,6 +116,11 @@ async def static_vector_tile_cache_asset( "-I", creation_options.implementation, ] + if creation_options.feature_filter: + command += ( + "--filter", + json.dumps(jsonable_encoder(creation_options.feature_filter)) + ) create_vector_tile_cache = TileCacheJob( dataset=dataset, diff --git a/app/utils/path.py b/app/utils/path.py index ad1cf06bc..9e51c7f10 100644 --- a/app/utils/path.py +++ b/app/utils/path.py @@ -69,7 +69,7 @@ def get_asset_uri( AssetType.cog: f"s3://{DATA_LAKE_BUCKET}/{dataset}/{version}/raster/{srid}/cog/{implementation}.tif", AssetType.raster_tile_cache: f"{TILE_CACHE_URL}/{dataset}/{version}/{implementation}/{{z}}/{{x}}/{{y}}.png", AssetType.shapefile: f"s3://{DATA_LAKE_BUCKET}/{dataset}/{version}/vector/{srid}/{dataset}_{version}.shp.zip", - AssetType.ndjson: f"s3://{DATA_LAKE_BUCKET}/{dataset}/{version}/vector/{srid}/{dataset}_{version}.ndjson", + AssetType.ndjson: f"s3://{DATA_LAKE_BUCKET}/{dataset}/{version}/vector/{srid}/{implementation}.ndjson", AssetType.grid_1x1: f"s3://{DATA_LAKE_BUCKET}/{dataset}/{version}/vector/{srid}/{dataset}_{version}_1x1.tsv", AssetType.geopackage: f"s3://{DATA_LAKE_BUCKET}/{dataset}/{version}/vector/{srid}/{dataset}_{version}.gpkg", AssetType.csv: f"s3://{DATA_LAKE_BUCKET}/{dataset}/{version}/text/{dataset}_{version}.csv", diff --git a/batch/scripts/create_vector_tile_cache.sh b/batch/scripts/create_vector_tile_cache.sh index 4fe2547e7..5c8d87ca4 100755 --- a/batch/scripts/create_vector_tile_cache.sh +++ b/batch/scripts/create_vector_tile_cache.sh @@ -10,19 +10,35 @@ set -e # -z | --max_zoom # -t | --tile_strategy # -I | --implementation + +# optional arguments +# --filter + ME=$(basename "$0") . get_arguments.sh "$@" -# Set TILE_STRATEGY + +NDJSON_FILE="data.json" + +# Build an array of arguments to pass to tippecanoe +TIPPE_ARG_ARRAY=( + "-e" "tilecache" + "-Z${MIN_ZOOM}" + "-z${MAX_ZOOM}" + "--preserve-input-order" + "-P" + "-n" "${DATASET}" +) + case ${TILE_STRATEGY} in discontinuous) # Discontinuous polygon features - STRATEGY=("--drop-densest-as-needed" "--extend-zooms-if-still-dropping") + TIPPE_ARG_ARRAY+=("--drop-densest-as-needed" "--extend-zooms-if-still-dropping") ;; continuous) # Continuous polygon features - STRATEGY=("--coalesce-densest-as-needed" "--extend-zooms-if-still-dropping") + TIPPE_ARG_ARRAY+=("--coalesce-densest-as-needed" "--extend-zooms-if-still-dropping") ;; keep_all) # never drop or coalesce feature, ignore size and feature count - STRATEGY=("-r1") + TIPPE_ARG_ARRAY+=("-r1") ;; *) echo "Invalid Tile Cache option -${TILE_STRATEGY}" @@ -30,11 +46,18 @@ keep_all) # never drop or coalesce feature, ignore size and feature count ;; esac -echo "Fetch NDJSON data from Data Lake ${SRC} -> ${DATASET}" -aws s3 cp "${SRC}" "${DATASET}" --no-progress +if [ -n "${FILTER}" ]; then + echo "${FILTER}" > feature_filter.txt + TIPPE_ARG_ARRAY+=("-J" "feature_filter.txt") +fi + +TIPPE_ARG_ARRAY+=("${NDJSON_FILE}") + +echo "Fetching NDJSON file from the Data Lake: ${SRC} -> ${NDJSON_FILE}..." +aws s3 cp "${SRC}" "${NDJSON_FILE}" --no-progress -echo "Build Tile Cache" -tippecanoe -Z"${MIN_ZOOM}" -z"${MAX_ZOOM}" -e tilecache "${STRATEGY[@]}" -P -n "${DATASET}" "${DATASET}" --preserve-input-order +echo "Building Tile Cache with Tippecanoe..." +tippecanoe "${TIPPE_ARG_ARRAY[@]}" -echo "Upload tiles to S3" +echo "Uploading tiles to S3 with TilePutty..." tileputty tilecache --bucket "${TILE_CACHE}" --dataset "${DATASET}" --version "${VERSION}" --implementation "${IMPLEMENTATION}" --cores "${NUM_PROCESSES}" \ No newline at end of file diff --git a/batch/scripts/get_arguments.sh b/batch/scripts/get_arguments.sh index 33653af3a..067bd24d8 100755 --- a/batch/scripts/get_arguments.sh +++ b/batch/scripts/get_arguments.sh @@ -90,6 +90,11 @@ do shift # past argument shift # past value ;; + --filter) + FILTER="$2" + shift + shift + ;; -F|--format) FORMAT="$2" shift # past argument