Upgrade to GDAL 3.8.5, future-proof for 3.9.0 (use venvs)

Dockerfile

@@ -4,12 +4,12 @@ FROM tiangolo/uvicorn-gunicorn-fastapi:python3.10-slim
 ARG ENV
 
 RUN apt-get update -y \
-    && apt-get install --no-install-recommends -y gcc libc-dev musl-dev \
-    postgresql-client libpq-dev make git jq \
+    && apt-get install --no-install-recommends -y gcc g++ libc-dev \
+        postgresql-client libpq-dev make git jq libgdal-dev \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-RUN pip install --upgrade pip && pip install pipenv==v2022.11.30
+RUN pip install --upgrade pip && pip install pipenv==2024.0.1
 #TODO move to pipfile when operational
 RUN pip install newrelic
 

Pipfile

@@ -14,13 +14,14 @@ geopandas = "*"  # Needed by pixetl in batch script test
 gfw_pixetl = {git = "https://github.com/wri/gfw_pixetl.git", ref = "develop"}
 moto = {version = "<5", extras = ["awslambda", "batch", "ec2", "s3", "secretsmanager"]}
 openapi_spec_validator = "*"
+pandas = "<2.2"  # Needed by pixetl in batch script test
 pre-commit = "*"
 pytest = "*"
 pytest-asyncio = "<0.19"
 pytest-cov = "*"
 pytest-timeout = "*"
 rasterio = "*"
-responses = "<0.16"
+responses = "*"
 retrying = "*"  # Needed by pixetl in batch script test
 
 [packages]
@@ -43,7 +44,7 @@ google-cloud-storage = "*"
 httpcore = "*"
 httpx = "*"
 httpx-auth = "*"
-numpy = "*"
+numpy = "<2.0"
 orjson = "*"
 packaging = "*"
 pendulum = "<3"

batch/gdal-python.dockerfile

@@ -1,4 +1,4 @@
-FROM globalforestwatch/data-api-gdal:v1.1.11
+FROM globalforestwatch/data-api-gdal:v1.2.0
 
 # Copy scripts
 COPY ./batch/scripts/ /opt/scripts/
@@ -11,8 +11,6 @@ RUN chmod +x -R /opt/python/
 ENV PATH="/opt/scripts:${PATH}"
 ENV PATH="/opt/python:${PATH}"
 
-RUN pip install earthengine-api
-
 ENV WORKDIR="/tmp"
 
 ENTRYPOINT ["/opt/scripts/report_status.sh"]

batch/pixetl.dockerfile

@@ -1,4 +1,4 @@
-FROM globalforestwatch/pixetl:v1.7.6
+FROM globalforestwatch/pixetl:v1.7.7
 
 # Copy scripts
 COPY ./batch/scripts/ /opt/scripts/

batch/postgresql-client.dockerfile

@@ -1,4 +1,4 @@
-FROM globalforestwatch/data-api-postgresql:1.0.2
+FROM globalforestwatch/data-api-postgresql:v1.1.0
 
 # Copy scripts
 COPY ./batch/scripts/ /opt/scripts/
@@ -13,4 +13,4 @@ ENV PATH="/opt/python:${PATH}"
 
 WORKDIR /tmp
 
-ENTRYPOINT ["/opt/scripts/report_status.sh"]
+ENTRYPOINT ["/opt/scripts/report_status.sh"]

batch/python/raster_tile_cache.py

@@ -22,6 +22,8 @@
         "NUM_PROCESSES", os.environ.get("CORES", multiprocessing.cpu_count())
     )
 )
+VENV_DIR = os.environ.get("VENV_DIR", "/.venv")
+
 LOGGER = get_logger(__name__)
 
 
@@ -75,7 +77,8 @@ def create_tiles(args: Tuple[Tuple[str, str], str, str, str, str, int, bool, int
             gdal2tiles = "16bpp_gdal2tiles.py"
 
         cmd: List[str] = [
-            gdal2tiles,
+            f"{VENV_DIR}/bin/python",
+            os.path.join("/opt/python", gdal2tiles),
             f"--zoom={zoom_level}",
             "--s_srs",
             "EPSG:3857",

batch/scripts/apply_colormap.sh

@@ -26,5 +26,4 @@ ARG_ARRAY+=("--source-uri" "${SRC}")
 
 ARG_ARRAY+=("--target-prefix" "${TARGET}")
 
-# Run apply_symbology.py with the array of arguments
 apply_colormap.py "${ARG_ARRAY[@]}"

batch/scripts/cogify.sh

@@ -45,5 +45,4 @@ if [ -n "$EXPORT_TO_GEE" ]; then
   export_to_gee.py --dataset "${DATASET}" --implementation "${IMPLEMENTATION}"
 fi
 
-set +x
-
+set +x

batch/scripts/report_status.sh

@@ -12,6 +12,9 @@ URL=${STATUS_URL}/${AWS_BATCH_JOB_ID}
 
 OUTPUT_FILE="/tmp/${AWS_BATCH_JOB_ID}_output.txt"
 
+# Source the virtualenv with all our Python packages
+. "${VENV_DIR}"/bin/activate
+
 # If this is not the first attempt, and previous attempts failed due to OOM,
 # reduce the NUM_PROCESSES value (thus increasing memory per process)
 if [[ -n $AWS_BATCH_JOB_ATTEMPT ]] && [[ $AWS_BATCH_JOB_ATTEMPT -gt 1 ]]; then

batch/tile_cache.dockerfile

@@ -1,4 +1,4 @@
-FROM globalforestwatch/data-api-tippecanoe:v1.3.0
+FROM globalforestwatch/data-api-tippecanoe:v1.3.1
 
 # Copy scripts
 COPY ./batch/scripts/ /opt/scripts/

docker-compose.dev.yml

@@ -61,7 +61,7 @@ services:
 
   database_12:
     container_name: gfw-data-api-database_12
-    image: postgis/postgis:12-2.5-alpine
+    image: postgis/postgis:12-3.3-alpine
     restart: on-failure
     ports:
       - 54320:5432

docker-compose.prod.yml

@@ -46,7 +46,7 @@ services:
 
   database:
     container_name: gfw-data-api-database
-    image: postgis/postgis:12-2.5-alpine
+    image: postgis/postgis:12-3.3-alpine
     restart: on-failure
     ports:
       - 54320:5432

docker-compose.test.yml

@@ -71,7 +71,7 @@ services:
 
   test_database_12:
     container_name: gfw-data-api-test-database-12
-    image: postgis/postgis:12-2.5-alpine
+    image: postgis/postgis:12-3.3-alpine
     ports:
       - 54321:5432
     environment:

tests/routes/test_geostore.py

@@ -96,7 +96,7 @@ async def test_dataset_version_geostore(async_client: AsyncClient, batch_client)
     ############################
 
     # This is the hash obtained by POSTing the sample GeoJSON with Postman
-    sample_geojson_hash = "41b67a74-4ea2-df3f-c3f3-d7131a645f9a"
+    sample_geojson_hash = "b9faa657-34c9-96d4-fce4-8bb8a1507cb3"
 
     # The geometry should be accessible via the geostore table
     async with ContextEngine("READ"):