diff --git a/Dockerfile b/Dockerfile index f6e52816..e16f0d8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ FROM tiangolo/uvicorn-gunicorn-fastapi:python3.10-slim +# Comment to trigger an image rebuild + # Optional build argument for different environments ARG ENV diff --git a/app/models/pydantic/jobs.py b/app/models/pydantic/jobs.py index d5f6bb45..29535800 100644 --- a/app/models/pydantic/jobs.py +++ b/app/models/pydantic/jobs.py @@ -145,7 +145,7 @@ class GDALCOGJob(Job): vcpus = 8 memory = 64000 num_processes = 8 - attempts = 2 + attempts = 5 attempt_duration_seconds = int(DEFAULT_JOB_DURATION * 1.5) diff --git a/batch/python/export_to_gee.py b/batch/python/export_to_gee.py index af3ab6a0..235fa0d6 100644 --- a/batch/python/export_to_gee.py +++ b/batch/python/export_to_gee.py @@ -1,5 +1,13 @@ #!/usr/bin/env python +# Required options: +# --dataset {dataset} +# --implementation {implementation} +# +# Assumes that the cog file to be uploaded is available at {implementation}.tif +# Uploads the cog file to GCS at {GCS_BUCKET}/{dataset}/{implementation}.tif and +# creates a GEE asset that links to that GCS URI. + import json import os @@ -35,7 +43,7 @@ def upload_cog_to_gcs(dataset, implementation): bucket = storage_client.bucket(GCS_BUCKET) blob = bucket.blob(f"{dataset}/{implementation}.tif") - blob.upload_from_filename("cog.tif") + blob.upload_from_filename(f"{implementation}.tif") return f"gs://{GCS_BUCKET}/{dataset}/{implementation}.tif" @@ -44,7 +52,7 @@ def create_cog_backed_asset(dataset, implementation, gcs_path, service_account): credentials = ee.ServiceAccountCredentials(service_account, GCS_CREDENTIALS_FILE) ee.Initialize(credentials) - # delete any existing asset with the same dataset/implementatio + # delete any existing asset with the same dataset/implementation try: ee.data.deleteAsset(f"projects/{EE_PROJECT}/assets/{dataset}/{implementation}") except ee.EEException: diff --git a/batch/scripts/cogify.sh b/batch/scripts/cogify.sh index 3fb2d40b..2d44efe8 100755 --- a/batch/scripts/cogify.sh +++ b/batch/scripts/cogify.sh @@ -38,7 +38,7 @@ else fi # convert to COG using existing overviews, this adds some additional layout optimizations -gdal_translate "${IMPLEMENTATION}_merged.tif" "${IMPLEMENTATION}.tif" -of COG -co COMPRESS=DEFLATE -co BLOCKSIZE="${BLOCK_SIZE}" -co BIGTIFF=IF_SAFER -co NUM_THREADS=ALL_CPUS -co OVERVIEWS=FORCE_USE_EXISTING --config GDAL_CACHEMAX 70% --config GDAL_NUM_THREADS ALL_CPUS +gdal_translate "${IMPLEMENTATION}_merged.tif" "${IMPLEMENTATION}.tif" -of COG -co COMPRESS=DEFLATE -co BLOCKSIZE="${BLOCK_SIZE}" -co BIGTIFF=IF_SAFER -co NUM_THREADS=ALL_CPUS -co OVERVIEWS=FORCE_USE_EXISTING -co SPARSE_OK=TRUE --config GDAL_CACHEMAX 70% --config GDAL_NUM_THREADS ALL_CPUS # upload to data lake aws s3 cp "${IMPLEMENTATION}.tif" "${TARGET}"