From 24c383416097b77c48530eef1b68fdc57830d16a Mon Sep 17 00:00:00 2001 From: Dan Scales Date: Mon, 15 Apr 2024 09:04:59 -0700 Subject: [PATCH] GTC-2708 Expire all tile cache files when deleting a dataset version We currently delete all datalake files, but have not been deleting the tile cache files in s3://gfw-tiles. The expire_s3_objects() function seems to work, since it is already used when deleting only a static vector tile cache or a raster tile cache. Now that expire_s3_objects is being called when we delete a version, I needed to add in a monkeypatch for expire_s3_objects() in create_vector_source_version() and generic_raster_version() for the tests_v2 tests. --- app/tasks/delete_assets.py | 2 +- tests_v2/conftest.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/tasks/delete_assets.py b/app/tasks/delete_assets.py index 4582bae15..2528d1a0c 100644 --- a/app/tasks/delete_assets.py +++ b/app/tasks/delete_assets.py @@ -11,7 +11,7 @@ async def delete_all_assets(dataset: str, version: str) -> None: await delete_database_table_asset(dataset, version) delete_s3_objects(DATA_LAKE_BUCKET, f"{dataset}/{version}/") - # expire_s3_objects(TILE_CACHE_BUCKET, f"{dataset}/{version}/") + expire_s3_objects(TILE_CACHE_BUCKET, f"{dataset}/{version}/") flush_cloudfront_cache(TILE_CACHE_CLOUDFRONT_ID, [f"/{dataset}/{version}/*"]) diff --git a/tests_v2/conftest.py b/tests_v2/conftest.py index 33fbb3a53..99c052914 100755 --- a/tests_v2/conftest.py +++ b/tests_v2/conftest.py @@ -189,6 +189,7 @@ async def create_vector_source_version( monkeypatch.setattr(batch, "submit_batch_job", batch_job_mock.submit_batch_job) monkeypatch.setattr(vector_source_assets, "is_zipped", bool_function_closure(False)) monkeypatch.setattr(delete_assets, "delete_s3_objects", int_function_closure(1)) + monkeypatch.setattr(delete_assets, "expire_s3_objects", dict_function_closure({})) monkeypatch.setattr(versions, "flush_cloudfront_cache", dict_function_closure({})) monkeypatch.setattr( delete_assets, "flush_cloudfront_cache", dict_function_closure({}) @@ -248,6 +249,7 @@ async def generic_raster_version( monkeypatch.setattr(versions, "_verify_source_file_access", void_coroutine) monkeypatch.setattr(batch, "submit_batch_job", batch_job_mock.submit_batch_job) monkeypatch.setattr(delete_assets, "delete_s3_objects", int_function_closure(1)) + monkeypatch.setattr(delete_assets, "expire_s3_objects", dict_function_closure({})) monkeypatch.setattr(raster_tile_set_assets, "get_extent", get_extent_mocked) monkeypatch.setattr( delete_assets, "flush_cloudfront_cache", dict_function_closure({})