Merge branch 'develop' into gtc-2908/save_cog_s3

wri · Jul 31, 2024 · 45a91c4 · 45a91c4
2 parents 4b83ab9 + fb6fad9
commit 45a91c4
Show file tree

Hide file tree

Showing 6 changed files with 55 additions and 12 deletions.
diff --git a/app/models/pydantic/creation_options.py b/app/models/pydantic/creation_options.py
@@ -399,6 +399,11 @@ class StaticVectorTileCacheCreationOptions(TileCacheBaseModel):
         "for vector tile caches. `source` and `source-layer` attributes must use `dataset` name."
         "Styling rules will be used in autogenerated root.json and preview.",
     )
+    feature_filter: Optional[Dict[str, Any]] = Field(
+        None,
+        description="Optional tippecanoe feature filter(s). Uses the syntax of "
+        "[Mapbox legacy filters](https://docs.mapbox.com/style-spec/reference/other/#other-filters)"
+    )
 
 
 class StaticVectorFileCreationOptions(StrictBaseModel):

diff --git a/app/tasks/raster_tile_cache_assets/raster_tile_cache_assets.py b/app/tasks/raster_tile_cache_assets/raster_tile_cache_assets.py
@@ -32,6 +32,7 @@
 
 from ...errors import RecordNotFoundError
 
+
 async def raster_tile_cache_asset(
     dataset: str,
     version: str,

diff --git a/app/tasks/static_vector_tile_cache_assets.py b/app/tasks/static_vector_tile_cache_assets.py
@@ -3,6 +3,8 @@
 from typing import Any, Dict, List, Optional
 from uuid import UUID
 
+from fastapi.encoders import jsonable_encoder
+
 from ..crud import assets, metadata
 from ..errors import RecordNotFoundError
 from ..models.orm.assets import Asset as ORMAsset
@@ -55,7 +57,9 @@ async def static_vector_tile_cache_asset(
     # Create NDJSON asset as side effect
     ############################
 
-    ndjson_uri = get_asset_uri(dataset, version, AssetType.ndjson)
+    ndjson_uri = get_asset_uri(
+        dataset, version, AssetType.ndjson, creation_options.dict()
+    )
 
     ndjson_asset: ORMAsset = await assets.create_asset(
         dataset,
@@ -77,7 +81,7 @@ async def static_vector_tile_cache_asset(
         "-v",
         version,
         "-f",
-        f"{dataset}_{version}.ndjson",
+        f"{creation_options.implementation}.ndjson",
         "-F",
         "GeoJSONSeq",
         "-T",
@@ -112,6 +116,11 @@ async def static_vector_tile_cache_asset(
         "-I",
         creation_options.implementation,
     ]
+    if creation_options.feature_filter:
+        command += (
+            "--filter",
+            json.dumps(jsonable_encoder(creation_options.feature_filter))
+        )
 
     create_vector_tile_cache = TileCacheJob(
         dataset=dataset,

diff --git a/app/utils/path.py b/app/utils/path.py
@@ -69,7 +69,7 @@ def get_asset_uri(
         AssetType.cog: f"s3://{DATA_LAKE_BUCKET}/{dataset}/{version}/raster/{srid}/cog/{implementation}.tif",
         AssetType.raster_tile_cache: f"{TILE_CACHE_URL}/{dataset}/{version}/{implementation}/{{z}}/{{x}}/{{y}}.png",
         AssetType.shapefile: f"s3://{DATA_LAKE_BUCKET}/{dataset}/{version}/vector/{srid}/{dataset}_{version}.shp.zip",
-        AssetType.ndjson: f"s3://{DATA_LAKE_BUCKET}/{dataset}/{version}/vector/{srid}/{dataset}_{version}.ndjson",
+        AssetType.ndjson: f"s3://{DATA_LAKE_BUCKET}/{dataset}/{version}/vector/{srid}/{implementation}.ndjson",
         AssetType.grid_1x1: f"s3://{DATA_LAKE_BUCKET}/{dataset}/{version}/vector/{srid}/{dataset}_{version}_1x1.tsv",
         AssetType.geopackage: f"s3://{DATA_LAKE_BUCKET}/{dataset}/{version}/vector/{srid}/{dataset}_{version}.gpkg",
         AssetType.csv: f"s3://{DATA_LAKE_BUCKET}/{dataset}/{version}/text/{dataset}_{version}.csv",

diff --git a/batch/scripts/create_vector_tile_cache.sh b/batch/scripts/create_vector_tile_cache.sh
@@ -10,31 +10,54 @@ set -e
 # -z | --max_zoom
 # -t | --tile_strategy
 # -I | --implementation
+
+# optional arguments
+# --filter
+
 ME=$(basename "$0")
 . get_arguments.sh "$@"
 
-# Set TILE_STRATEGY
+
+NDJSON_FILE="data.json"
+
+# Build an array of arguments to pass to tippecanoe
+TIPPE_ARG_ARRAY=(
+  "-e" "tilecache"
+  "-Z${MIN_ZOOM}"
+  "-z${MAX_ZOOM}"
+  "--preserve-input-order"
+  "-P"
+  "-n" "${DATASET}"
+)
+
 case ${TILE_STRATEGY} in
 discontinuous) # Discontinuous polygon features
-  STRATEGY=("--drop-densest-as-needed" "--extend-zooms-if-still-dropping")
+  TIPPE_ARG_ARRAY+=("--drop-densest-as-needed" "--extend-zooms-if-still-dropping")
   ;;
 continuous) # Continuous polygon features
-  STRATEGY=("--coalesce-densest-as-needed" "--extend-zooms-if-still-dropping")
+  TIPPE_ARG_ARRAY+=("--coalesce-densest-as-needed" "--extend-zooms-if-still-dropping")
   ;;
 keep_all) # never drop or coalesce feature, ignore size and feature count
-  STRATEGY=("-r1")
+  TIPPE_ARG_ARRAY+=("-r1")
   ;;
 *)
   echo "Invalid Tile Cache option -${TILE_STRATEGY}"
   exit 1
   ;;
 esac
 
-echo "Fetch NDJSON data from Data Lake ${SRC} -> ${DATASET}"
-aws s3 cp "${SRC}" "${DATASET}" --no-progress
+if [ -n "${FILTER}" ]; then
+  echo "${FILTER}" > feature_filter.txt
+  TIPPE_ARG_ARRAY+=("-J" "feature_filter.txt")
+fi
+
+TIPPE_ARG_ARRAY+=("${NDJSON_FILE}")
+
+echo "Fetching NDJSON file from the Data Lake: ${SRC} -> ${NDJSON_FILE}..."
+aws s3 cp "${SRC}" "${NDJSON_FILE}" --no-progress
 
-echo "Build Tile Cache"
-tippecanoe -Z"${MIN_ZOOM}" -z"${MAX_ZOOM}" -e tilecache "${STRATEGY[@]}" -P -n "${DATASET}" "${DATASET}" --preserve-input-order
+echo "Building Tile Cache with Tippecanoe..."
+tippecanoe "${TIPPE_ARG_ARRAY[@]}"
 
-echo "Upload tiles to S3"
+echo "Uploading tiles to S3 with TilePutty..."
 tileputty tilecache --bucket "${TILE_CACHE}" --dataset "${DATASET}" --version "${VERSION}" --implementation "${IMPLEMENTATION}" --cores "${NUM_PROCESSES}"
diff --git a/batch/scripts/get_arguments.sh b/batch/scripts/get_arguments.sh
@@ -90,6 +90,11 @@ do
       shift # past argument
       shift # past value
       ;;
+      --filter)
+      FILTER="$2"
+      shift
+      shift
+      ;;
       -F|--format)
       FORMAT="$2"
       shift # past argument