Skip to content

Commit

Permalink
GTC-2708 Expire all tile cache files when deleting a dataset version
Browse files Browse the repository at this point in the history
We currently delete all datalake files, but have not been deleting the
tile cache files in s3://gfw-tiles. The expire_s3_objects() function
seems to work, since it is already used when deleting only a static
vector tile cache or a raster tile cache.

Now that expire_s3_objects is being called when we delete a version, I
needed to add in a monkeypatch for expire_s3_objects() in
create_vector_source_version() and generic_raster_version() for the
tests_v2 tests.
  • Loading branch information
danscales authored and manukala6 committed May 15, 2024
1 parent 86b1e26 commit b05b0ea
Show file tree
Hide file tree
Showing 2 changed files with 3 additions and 1 deletion.
2 changes: 1 addition & 1 deletion app/tasks/delete_assets.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
async def delete_all_assets(dataset: str, version: str) -> None:
await delete_database_table_asset(dataset, version)
delete_s3_objects(DATA_LAKE_BUCKET, f"{dataset}/{version}/")
# expire_s3_objects(TILE_CACHE_BUCKET, f"{dataset}/{version}/")
expire_s3_objects(TILE_CACHE_BUCKET, f"{dataset}/{version}/")
flush_cloudfront_cache(TILE_CACHE_CLOUDFRONT_ID, [f"/{dataset}/{version}/*"])


Expand Down
2 changes: 2 additions & 0 deletions tests_v2/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@ async def create_vector_source_version(
monkeypatch.setattr(batch, "submit_batch_job", batch_job_mock.submit_batch_job)
monkeypatch.setattr(vector_source_assets, "is_zipped", bool_function_closure(False))
monkeypatch.setattr(delete_assets, "delete_s3_objects", int_function_closure(1))
monkeypatch.setattr(delete_assets, "expire_s3_objects", dict_function_closure({}))
monkeypatch.setattr(versions, "flush_cloudfront_cache", dict_function_closure({}))
monkeypatch.setattr(
delete_assets, "flush_cloudfront_cache", dict_function_closure({})
Expand Down Expand Up @@ -248,6 +249,7 @@ async def generic_raster_version(
monkeypatch.setattr(versions, "_verify_source_file_access", void_coroutine)
monkeypatch.setattr(batch, "submit_batch_job", batch_job_mock.submit_batch_job)
monkeypatch.setattr(delete_assets, "delete_s3_objects", int_function_closure(1))
monkeypatch.setattr(delete_assets, "expire_s3_objects", dict_function_closure({}))
monkeypatch.setattr(raster_tile_set_assets, "get_extent", get_extent_mocked)
monkeypatch.setattr(
delete_assets, "flush_cloudfront_cache", dict_function_closure({})
Expand Down

0 comments on commit b05b0ea

Please sign in to comment.