feat: optionally perform filtering

wri · Jul 17, 2024 · 1ef0215 · 1ef0215
1 parent ffa83a7
commit 1ef0215
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 4 deletions.
diff --git a/app/tasks/static_vector_tile_cache_assets.py b/app/tasks/static_vector_tile_cache_assets.py
@@ -55,7 +55,9 @@ async def static_vector_tile_cache_asset(
     # Create NDJSON asset as side effect
     ############################
 
-    ndjson_uri = get_asset_uri(dataset, version, AssetType.ndjson, creation_options.dict())
+    ndjson_uri = get_asset_uri(
+        dataset, version, AssetType.ndjson, creation_options.dict()
+    )
 
     ndjson_asset: ORMAsset = await assets.create_asset(
         dataset,
@@ -114,6 +116,10 @@ async def static_vector_tile_cache_asset(
                 creation_options.tile_strategy,
                 "-I",
                 creation_options.implementation,
+                "--where_field",
+                where_filter.field,
+                "--where_values",
+                ",".join(where_filter.values_in),
             ]
             tile_cache_jobs.append(
                 TileCacheJob(

diff --git a/batch/scripts/create_vector_tile_cache.sh b/batch/scripts/create_vector_tile_cache.sh
@@ -30,11 +30,19 @@ keep_all) # never drop or coalesce feature, ignore size and feature count
   ;;
 esac
 
-echo "Fetch NDJSON data from Data Lake ${SRC} -> ${DATASET}"
-aws s3 cp "${SRC}" "${DATASET}" --no-progress
+echo "Fetch NDJSON data from Data Lake ${SRC} -> 'data.ndjson'"
+aws s3 cp "${SRC}" 'data.ndjson' --no-progress
+
+FINAL_DATA='data.ndjson'
+
+if [ -n "$WHERE_FIELD" ]; then
+  FINAL_DATA='filtered_data.ndjson'
+  echo "Perform Filtering"
+  ogr2ogr "${FINAL_DATA}" 'data.ndjson' -where "${WHERE_FIELD} IN (${WHERE_VALUES})"
+fi
 
 echo "Build Tile Cache"
-tippecanoe -Z"${MIN_ZOOM}" -z"${MAX_ZOOM}" -e tilecache "${STRATEGY[@]}" -P -n "${DATASET}" "${DATASET}" --preserve-input-order
+tippecanoe -Z"${MIN_ZOOM}" -z"${MAX_ZOOM}" -e tilecache "${STRATEGY[@]}" -P -n "${DATASET}" "${FINAL_DATA}" --preserve-input-order
 
 echo "Upload tiles to S3"
 tileputty tilecache --bucket "${TILE_CACHE}" --dataset "${DATASET}" --version "${VERSION}" --implementation "${IMPLEMENTATION}" --cores "${NUM_PROCESSES}"
diff --git a/batch/scripts/get_arguments.sh b/batch/scripts/get_arguments.sh
@@ -241,6 +241,16 @@ do
       shift # past argument
       shift # past value
       ;;
+      --where_field)
+      WHERE_FIELD="$2"
+      shift
+      shift
+      ;;
+      --where_values)
+      WHERE_VALUES="$2"
+      shift
+      shift
+      ;;
 
       *)    # unknown option
       POSITIONAL+=("$1") # save it in an array for later