From 201bc8b66387e5a922529c8aceb5824e294f4a2b Mon Sep 17 00:00:00 2001 From: Daniel Mannarino Date: Wed, 10 Jan 2024 23:27:39 -0500 Subject: [PATCH 1/5] Make get_field_attributes return fields in the order specified --- app/utils/fields.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/app/utils/fields.py b/app/utils/fields.py index d6e4dd74a..783ee2ffd 100644 --- a/app/utils/fields.py +++ b/app/utils/fields.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List +from typing import Any, Dict, List, Set from ..crud import assets, metadata as metadata_crud from ..models.orm.assets import Asset as ORMAsset @@ -8,28 +8,30 @@ async def get_field_attributes( dataset: str, version: str, creation_options: CreationOptions ) -> List[Dict[str, Any]]: - """Get field attribute list from creation options. - - If no attribute list provided, use all fields from DB table, marked - as `is_feature_info`. Otherwise compare to provide list with - available fields and use intersection. + """Get list of field attributes on the asset which are marked as `is_feature_info` + If a field list is provided in creation options, limit the list to those provided, + in the order provided. Invalid provided fields are silently ignored. """ default_asset: ORMAsset = await assets.get_default_asset(dataset, version) - fields = await metadata_crud.get_asset_fields_dicts(default_asset) + asset_fields = await metadata_crud.get_asset_fields_dicts(default_asset) - field_attributes: List[Dict[str, Any]] = [ - field for field in fields if field["is_feature_info"] - ] + name_to_feature_fields: Dict[str, Dict] = { + field["name"]: field + for field in asset_fields + if field["is_feature_info"] + } if ( "field_attributes" in creation_options.__fields__ and creation_options.field_attributes ): - field_attributes = [ - field - for field in field_attributes - if field["name"] in creation_options.field_attributes + asset_field_attributes = [ + name_to_feature_fields[field_name] + for field_name in creation_options.field_attributes + if field_name in name_to_feature_fields ] + else: + asset_field_attributes = list(name_to_feature_fields.values()) - return field_attributes + return asset_field_attributes From 920928506c51e0fbd83c5fb997bbab936705729b Mon Sep 17 00:00:00 2001 From: Daniel Mannarino Date: Wed, 10 Jan 2024 23:28:42 -0500 Subject: [PATCH 2/5] Add the option to include the tile ID in a 1x1 export --- app/models/pydantic/creation_options.py | 10 +++++++++- app/tasks/static_vector_1x1_assets.py | 2 ++ batch/python/export_1x1_grid.py | 24 ++++++++++++++++++++---- batch/scripts/export_1x1_grid.sh | 7 +++++-- batch/scripts/get_arguments.sh | 5 +++++ 5 files changed, 41 insertions(+), 7 deletions(-) diff --git a/app/models/pydantic/creation_options.py b/app/models/pydantic/creation_options.py index 44fb25659..2ab424847 100644 --- a/app/models/pydantic/creation_options.py +++ b/app/models/pydantic/creation_options.py @@ -380,6 +380,13 @@ class StaticVectorFileCreationOptions(StrictBaseModel): ) +class StaticVector1x1CreationOptions(StaticVectorFileCreationOptions): + include_tile_id: Optional[bool] = Field( + False, + description="Whether or not to include the tile_id of each feature" + ) + + SourceCreationOptions = Union[ TableSourceCreationOptions, RasterTileSetSourceCreationOptions, @@ -390,6 +397,7 @@ class StaticVectorFileCreationOptions(StrictBaseModel): TableAssetCreationOptions, RasterTileCacheCreationOptions, StaticVectorTileCacheCreationOptions, + StaticVector1x1CreationOptions, StaticVectorFileCreationOptions, DynamicVectorTileCacheCreationOptions, RasterTileSetAssetCreationOptions, @@ -412,7 +420,7 @@ class CreationOptionsResponse(Response): AssetType.dynamic_vector_tile_cache: DynamicVectorTileCacheCreationOptions, AssetType.static_vector_tile_cache: StaticVectorTileCacheCreationOptions, AssetType.ndjson: StaticVectorFileCreationOptions, - AssetType.grid_1x1: StaticVectorFileCreationOptions, + AssetType.grid_1x1: StaticVector1x1CreationOptions, AssetType.shapefile: StaticVectorFileCreationOptions, AssetType.geopackage: StaticVectorFileCreationOptions, AssetType.raster_tile_set: RasterTileSetAssetCreationOptions, diff --git a/app/tasks/static_vector_1x1_assets.py b/app/tasks/static_vector_1x1_assets.py index 303bed869..55e8c444f 100644 --- a/app/tasks/static_vector_1x1_assets.py +++ b/app/tasks/static_vector_1x1_assets.py @@ -53,6 +53,8 @@ async def static_vector_1x1_asset( version, "-C", ",".join([field["name"] for field in field_attributes]), + "--include_tile_id", + str(creation_options.include_tile_id), "-T", grid_1x1_uri, ] diff --git a/batch/python/export_1x1_grid.py b/batch/python/export_1x1_grid.py index 6a2beef03..f7759dfff 100644 --- a/batch/python/export_1x1_grid.py +++ b/batch/python/export_1x1_grid.py @@ -356,7 +356,7 @@ def src_table(dataset: str, version: str) -> Table: def get_sql( - dataset: str, version: str, fields: List[str], grid_id: str, tcl: bool, glad: bool + dataset: str, version: str, fields: List[str], include_tile_id: bool, grid_id: str, tcl: bool, glad: bool ) -> Select: """Generate SQL statement.""" @@ -366,6 +366,9 @@ def get_sql( nested_columns = [field.split(",") for field in fields] columns = [column(c) for columns in nested_columns for c in columns] + if include_tile_id: + columns.append(literal_column(f"'{grid_id}'").label("tile_id")) + sql: Select = ( select(columns + [tcl_column, glad_column, geom_column]) .select_from(src_table(dataset, version).alias("t")) @@ -379,7 +382,7 @@ def get_sql( async def run( - loop: AbstractEventLoop, dataset: str, version: str, fields: List[str] + loop: AbstractEventLoop, dataset: str, version: str, fields: List[str], include_tile_id: bool ) -> None: async def copy_tiles(i: int, tile: Tuple[str, bool, bool]) -> None: if i == 0: @@ -406,7 +409,17 @@ async def copy_tiles(i: int, tile: Tuple[str, bool, bool]) -> None: password=PGPASSWORD, ) result = await con.copy_from_query( - str(get_sql(dataset, version, fields, grid_id, tcl, glad)), + str( + get_sql( + dataset, + version, + fields, + include_tile_id, + grid_id, + tcl, + glad + ) + ), output=output, format="csv", delimiter="\t", @@ -445,6 +458,9 @@ async def copy_tiles(i: int, tile: Tuple[str, bool, bool]) -> None: parser.add_argument( "--column_names", "-C", type=str, nargs="+", help="Column names to include" ) + parser.add_argument( + "--include_tile_id", type=bool, default=False, help="Include tile_id in the output" + ) args = parser.parse_args() loop: AbstractEventLoop = asyncio.get_event_loop() - loop.run_until_complete(run(loop, args.dataset, args.version, args.column_names)) + loop.run_until_complete(run(loop, args.dataset, args.version, args.column_names, args.include_tile_id)) diff --git a/batch/scripts/export_1x1_grid.sh b/batch/scripts/export_1x1_grid.sh index 6831e232c..3e75bebf9 100755 --- a/batch/scripts/export_1x1_grid.sh +++ b/batch/scripts/export_1x1_grid.sh @@ -2,17 +2,20 @@ set -e -# requires arguments +# required arguments # -d | --dataset # -v | --version # -C | --column_names # -T | --target +# +# optional arguments +# --include_tile_id ME=$(basename "$0") . get_arguments.sh "$@" echo "PYTHON: Create 1x1 grid files" -export_1x1_grid.py -d "$DATASET" -v "$VERSION" -C "$COLUMN_NAMES" +export_1x1_grid.py -d "$DATASET" -v "$VERSION" -C "$COLUMN_NAMES" --include_tile_id "$INCLUDE_TILE_ID" echo "Combine output files" echo ./*.tmp | xargs cat >> "${DATASET}_${VERSION}_1x1.tsv" diff --git a/batch/scripts/get_arguments.sh b/batch/scripts/get_arguments.sh index 297e5eab5..51605f95d 100755 --- a/batch/scripts/get_arguments.sh +++ b/batch/scripts/get_arguments.sh @@ -105,6 +105,11 @@ do shift # past argument shift # past value ;; + --include_tile_id) + INCLUDE_TILE_ID="$2" + shift # past argument + shift # past value + ;; -j|--json) JSON="$2" shift # past argument From 85d620fdfea592b85a4a915acff7264e1d5cf14b Mon Sep 17 00:00:00 2001 From: Daniel Mannarino Date: Thu, 11 Jan 2024 15:56:39 -0500 Subject: [PATCH 3/5] Adjust order of creation options in union to fix validation error --- app/models/pydantic/creation_options.py | 2 +- app/tasks/static_vector_1x1_assets.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/pydantic/creation_options.py b/app/models/pydantic/creation_options.py index 2ab424847..fb82d59ba 100644 --- a/app/models/pydantic/creation_options.py +++ b/app/models/pydantic/creation_options.py @@ -397,8 +397,8 @@ class StaticVector1x1CreationOptions(StaticVectorFileCreationOptions): TableAssetCreationOptions, RasterTileCacheCreationOptions, StaticVectorTileCacheCreationOptions, - StaticVector1x1CreationOptions, StaticVectorFileCreationOptions, + StaticVector1x1CreationOptions, DynamicVectorTileCacheCreationOptions, RasterTileSetAssetCreationOptions, ] diff --git a/app/tasks/static_vector_1x1_assets.py b/app/tasks/static_vector_1x1_assets.py index 55e8c444f..7df14a7ba 100644 --- a/app/tasks/static_vector_1x1_assets.py +++ b/app/tasks/static_vector_1x1_assets.py @@ -19,7 +19,7 @@ async def static_vector_1x1_asset( asset_id: UUID, input_data: Dict[str, Any], ) -> ChangeLog: - """Create Vector tile cache and NDJSON file as intermediate data.""" + """Export a TSV to S3 with features in a 1x1 grid of tiles.""" ####################### # Update asset metadata From 93fbf676a0d1790d552d24eed8618e1af5b8b00b Mon Sep 17 00:00:00 2001 From: Daniel Mannarino Date: Fri, 12 Jan 2024 00:27:10 -0500 Subject: [PATCH 4/5] Adjust imports in fields.py to ease testing; add tests for get_field_attributes --- app/utils/fields.py | 9 +-- tests_v2/unit/app/utils/test_fields.py | 92 ++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 tests_v2/unit/app/utils/test_fields.py diff --git a/app/utils/fields.py b/app/utils/fields.py index 783ee2ffd..d1f2ff725 100644 --- a/app/utils/fields.py +++ b/app/utils/fields.py @@ -1,6 +1,7 @@ -from typing import Any, Dict, List, Set +from typing import Any, Dict, List -from ..crud import assets, metadata as metadata_crud +from ..crud.assets import get_default_asset +from ..crud.metadata import get_asset_fields_dicts from ..models.orm.assets import Asset as ORMAsset from ..models.pydantic.creation_options import CreationOptions @@ -13,8 +14,8 @@ async def get_field_attributes( in the order provided. Invalid provided fields are silently ignored. """ - default_asset: ORMAsset = await assets.get_default_asset(dataset, version) - asset_fields = await metadata_crud.get_asset_fields_dicts(default_asset) + default_asset: ORMAsset = await get_default_asset(dataset, version) + asset_fields = await get_asset_fields_dicts(default_asset) name_to_feature_fields: Dict[str, Dict] = { field["name"]: field diff --git a/tests_v2/unit/app/utils/test_fields.py b/tests_v2/unit/app/utils/test_fields.py new file mode 100644 index 000000000..c3f27e4da --- /dev/null +++ b/tests_v2/unit/app/utils/test_fields.py @@ -0,0 +1,92 @@ +from unittest.mock import AsyncMock + +import pytest +from _pytest.monkeypatch import MonkeyPatch + +from app.crud.assets import get_default_asset +from app.models.pydantic.creation_options import StaticVector1x1CreationOptions +from app.utils import fields + + +@pytest.mark.asyncio +async def test_get_field_attributes_no_specified_fields(monkeypatch: MonkeyPatch): + creation_options = {} + + mock_get_default_asset = AsyncMock(get_default_asset) + monkeypatch.setattr(fields, "get_default_asset", mock_get_default_asset) + + mock_get_asset_fields_dicts = AsyncMock(get_default_asset) + mock_get_asset_fields_dicts.return_value = [ + {"name": "something_wacky", "is_feature_info": True}, + {"name": "gid_2", "is_feature_info": True}, + {"name": "not_feature_field", "is_feature_info": False}, + {"name": "something_else", "is_feature_info": True}, + ] + monkeypatch.setattr(fields, "get_asset_fields_dicts", mock_get_asset_fields_dicts) + + foo = await fields.get_field_attributes("some_dataset", "v1.5", StaticVector1x1CreationOptions(**creation_options)) + assert foo == [ + {"name": "something_wacky", "is_feature_info": True}, + {"name": "gid_2", "is_feature_info": True}, + {"name": "something_else", "is_feature_info": True}, + ] + + +@pytest.mark.asyncio +async def test_get_field_attributes_respects_requested_order_1(monkeypatch: MonkeyPatch): + creation_options = { + "include_tile_id": True, + "field_attributes": [ + "gfw_geostore_id", + "gid_0", + "gid_1", + "gid_2" + ] + } + + mock_get_default_asset = AsyncMock(get_default_asset) + monkeypatch.setattr(fields, "get_default_asset", mock_get_default_asset) + + mock_get_asset_fields_dicts = AsyncMock(get_default_asset) + mock_get_asset_fields_dicts.return_value = [ + {"name": "something_wacky", "is_feature_info": True}, + {"name": "gid_2", "is_feature_info": True}, + {"name": "gid_0", "is_feature_info": True}, + ] + monkeypatch.setattr(fields, "get_asset_fields_dicts", mock_get_asset_fields_dicts) + + foo = await fields.get_field_attributes("some_dataset", "v1.5", StaticVector1x1CreationOptions(**creation_options)) + assert foo == [ + {"name": "gid_0", "is_feature_info": True}, + {"name": "gid_2", "is_feature_info": True}, + ] + + +@pytest.mark.asyncio +async def test_get_field_attributes_respects_requested_order_2(monkeypatch: MonkeyPatch): + creation_options = { + "include_tile_id": True, + "field_attributes": [ + "gfw_geostore_id", + "gid_0", + "gid_1", + "gid_2" + ] + } + + mock_get_default_asset = AsyncMock(get_default_asset) + monkeypatch.setattr(fields, "get_default_asset", mock_get_default_asset) + + mock_get_asset_fields_dicts = AsyncMock(get_default_asset) + mock_get_asset_fields_dicts.return_value = [ + {"name": "something_wacky", "is_feature_info": True}, + {"name": "gid_2", "is_feature_info": True}, + {"name": "gid_0", "is_feature_info": True}, + ] + monkeypatch.setattr(fields, "get_asset_fields_dicts", mock_get_asset_fields_dicts) + + foo = await fields.get_field_attributes("some_dataset", "v1.5", StaticVector1x1CreationOptions(**creation_options)) + assert foo == [ + {"name": "gid_0", "is_feature_info": True}, + {"name": "gid_2", "is_feature_info": True}, + ] From 63c0ab5615ea64a070da068cc5c0cf7d1ba5c269 Mon Sep 17 00:00:00 2001 From: Daniel Mannarino Date: Fri, 12 Jan 2024 02:31:56 -0500 Subject: [PATCH 5/5] Add tests; fix bug found by tests --- app/tasks/static_vector_1x1_assets.py | 5 +- batch/python/export_1x1_grid.py | 2 +- batch/scripts/export_1x1_grid.sh | 9 +- batch/scripts/get_arguments.sh | 3 +- tests/tasks/test_vector_tile_assets.py | 222 +++++++++++++++++++------ 5 files changed, 185 insertions(+), 56 deletions(-) diff --git a/app/tasks/static_vector_1x1_assets.py b/app/tasks/static_vector_1x1_assets.py index 7df14a7ba..a3ad84e69 100644 --- a/app/tasks/static_vector_1x1_assets.py +++ b/app/tasks/static_vector_1x1_assets.py @@ -53,12 +53,13 @@ async def static_vector_1x1_asset( version, "-C", ",".join([field["name"] for field in field_attributes]), - "--include_tile_id", - str(creation_options.include_tile_id), "-T", grid_1x1_uri, ] + if creation_options.include_tile_id: + command.append("--include_tile_id") + export_1x1_grid = PostgresqlClientJob( dataset=dataset, job_name="export_1x1_grid", diff --git a/batch/python/export_1x1_grid.py b/batch/python/export_1x1_grid.py index f7759dfff..690986f9e 100644 --- a/batch/python/export_1x1_grid.py +++ b/batch/python/export_1x1_grid.py @@ -459,7 +459,7 @@ async def copy_tiles(i: int, tile: Tuple[str, bool, bool]) -> None: "--column_names", "-C", type=str, nargs="+", help="Column names to include" ) parser.add_argument( - "--include_tile_id", type=bool, default=False, help="Include tile_id in the output" + "--include_tile_id", action='store_true', help="Include tile_id in the output" ) args = parser.parse_args() loop: AbstractEventLoop = asyncio.get_event_loop() diff --git a/batch/scripts/export_1x1_grid.sh b/batch/scripts/export_1x1_grid.sh index 3e75bebf9..00044ecb4 100755 --- a/batch/scripts/export_1x1_grid.sh +++ b/batch/scripts/export_1x1_grid.sh @@ -15,7 +15,14 @@ ME=$(basename "$0") . get_arguments.sh "$@" echo "PYTHON: Create 1x1 grid files" -export_1x1_grid.py -d "$DATASET" -v "$VERSION" -C "$COLUMN_NAMES" --include_tile_id "$INCLUDE_TILE_ID" +ARG_ARRAY=("--dataset" "${DATASET}" + "--version" "${VERSION}" + "-C" "${COLUMN_NAMES}") + +if [ -n "${INCLUDE_TILE_ID}" ]; then + ARG_ARRAY+=("--include_tile_id") +fi +export_1x1_grid.py "${ARG_ARRAY[@]}" echo "Combine output files" echo ./*.tmp | xargs cat >> "${DATASET}_${VERSION}_1x1.tsv" diff --git a/batch/scripts/get_arguments.sh b/batch/scripts/get_arguments.sh index 51605f95d..7b69d3596 100755 --- a/batch/scripts/get_arguments.sh +++ b/batch/scripts/get_arguments.sh @@ -106,9 +106,8 @@ do shift # past value ;; --include_tile_id) - INCLUDE_TILE_ID="$2" + INCLUDE_TILE_ID="TRUE" shift # past argument - shift # past value ;; -j|--json) JSON="$2" diff --git a/tests/tasks/test_vector_tile_assets.py b/tests/tasks/test_vector_tile_assets.py index 046a7e545..5fcde34a3 100644 --- a/tests/tasks/test_vector_tile_assets.py +++ b/tests/tasks/test_vector_tile_assets.py @@ -1,3 +1,4 @@ +import csv import json from unittest.mock import patch from urllib.parse import urlparse @@ -130,56 +131,6 @@ async def test_vector_tile_asset( ) assert resp["KeyCount"] == 0 - ########### - # 1x1 Grid - ########### - ### Create static tile cache asset - httpx.delete(f"http://localhost:{PORT}") - - input_data = { - "asset_type": "1x1 grid", - "is_managed": True, - "creation_options": {}, - } - - response = await async_client.post( - f"/dataset/{dataset}/{version}/assets", json=input_data - ) - assert response.status_code == 202 - asset_id = response.json()["data"]["asset_id"] - - # get tasks id from change log and wait until finished - response = await async_client.get(f"/asset/{asset_id}/change_log") - - assert response.status_code == 200 - tasks = json.loads(response.json()["data"][-1]["detail"]) - task_ids = [task["job_id"] for task in tasks] - - # make sure, all jobs completed - status = await poll_jobs(task_ids, logs=logs, async_client=async_client) - assert status == "saved" - - response = await async_client.get(f"/dataset/{dataset}/{version}/assets") - assert response.status_code == 200 - - # there should be 4 assets now (geodatabase table, dynamic vector tile cache and static vector tile cache (already deleted ndjson) - assert len(response.json()["data"]) == 4 - - # Check if file is in tile cache - resp = s3_client.list_objects_v2( - Bucket=DATA_LAKE_BUCKET, Prefix=f"{dataset}/{version}/vector/" - ) - assert resp["KeyCount"] == 1 - - response = await async_client.delete(f"/asset/{asset_id}") - assert response.status_code == 200 - - # Check if file was deleted - resp = s3_client.list_objects_v2( - Bucket=DATA_LAKE_BUCKET, Prefix=f"{dataset}/{version}/vector/" - ) - assert resp["KeyCount"] == 0 - ########### # Vector file export ########### @@ -250,3 +201,174 @@ async def test_vector_tile_asset( ) assert response.status_code == 200 assert mocked_cloudfront_client.called + + +@pytest.mark.asyncio +async def test_vector_tile_asset_1x1_grid( + batch_client, async_client: AsyncClient, monkeypatch +): + _, logs = batch_client + + ############################ + # Setup test + ############################ + + dataset = "test" + + version = "v1.1.1" + input_data = { + "creation_options": { + "source_type": "vector", + "source_uri": [f"s3://{BUCKET}/{SHP_NAME}"], + "source_driver": "ESRI Shapefile", + "create_dynamic_vector_tile_cache": False, + }, + } + + await create_default_asset( + dataset, + version, + version_payload=input_data, + async_client=async_client, + logs=logs, + execute_batch_jobs=True, + skip_dataset=False, + ) + + httpx.delete(f"http://localhost:{PORT}") + + ########### + # 1x1 Grid + ########### + input_data = { + "asset_type": "1x1 grid", + "creation_options": { + "field_attributes": [ + "gfw_geostore_id" + ] + }, + } + + response = await async_client.post( + f"/dataset/{dataset}/{version}/assets", json=input_data + ) + assert response.status_code == 202 + asset_id = response.json()["data"]["asset_id"] + + # Get task ids from change log and wait until finished + changelog_response = await async_client.get(f"/asset/{asset_id}/change_log") + tasks_dict = json.loads(changelog_response.json()["data"][-1]["detail"]) + task_ids = [task["job_id"] for task in tasks_dict] + + # Make sure all jobs completed + status = await poll_jobs(task_ids, logs=logs, async_client=async_client) + assert status == "saved" + + # There should be 2 assets now (geodatabase table and 1x1 TSV) + assets_response = await async_client.get(f"/dataset/{dataset}/{version}/assets") + assert len(assets_response.json()["data"]) == 2 + + # Verify the TSV was created + s3_client = get_s3_client() + expected_prefix = f"{dataset}/{version}/vector/epsg-4326" + resp = s3_client.list_objects_v2( + Bucket=DATA_LAKE_BUCKET, Prefix=expected_prefix + ) + assert resp["KeyCount"] == 1 + + # Sanity-check the TSV + s3_client.download_file( + DATA_LAKE_BUCKET, + f"{expected_prefix}/{dataset}_{version}_1x1.tsv", + "/tmp/1x1.tsv" + ) + with open("/tmp/1x1.tsv", "r") as f: + reader = csv.reader(f, dialect=csv.excel_tab) + header = next(reader) + assert header == ["gfw_geostore_id", 'tcl', 'glad', 'geom'] + + assert next(reader, None) is not None + + # Make sure deleting the asset deletes the TSV + response = await async_client.delete(f"/asset/{asset_id}") + assert response.status_code == 200 + + resp = s3_client.list_objects_v2( + Bucket=DATA_LAKE_BUCKET, Prefix=expected_prefix + ) + assert resp["KeyCount"] == 0 + + +@pytest.mark.asyncio +async def test_vector_tile_asset_1x1_grid_include_tile_id( + batch_client, async_client: AsyncClient, monkeypatch +): + _, logs = batch_client + + ############################ + # Setup test + ############################ + + dataset = "test" + + version = "v1.1.1" + input_data = { + "creation_options": { + "source_type": "vector", + "source_uri": [f"s3://{BUCKET}/{SHP_NAME}"], + "source_driver": "ESRI Shapefile", + "create_dynamic_vector_tile_cache": False, + }, + } + + await create_default_asset( + dataset, + version, + version_payload=input_data, + async_client=async_client, + logs=logs, + execute_batch_jobs=True, + skip_dataset=False, + ) + + httpx.delete(f"http://localhost:{PORT}") + + ########### + # 1x1 Grid + ########### + input_data = { + "asset_type": "1x1 grid", + "creation_options": { + "include_tile_id": True, + "field_attributes": [ + "gfw_geostore_id" + ] + }, + } + + response = await async_client.post( + f"/dataset/{dataset}/{version}/assets", json=input_data + ) + assert response.status_code == 202 + asset_id = response.json()["data"]["asset_id"] + + # Get task ids from change log and wait until finished + changelog_response = await async_client.get(f"/asset/{asset_id}/change_log") + tasks_dict = json.loads(changelog_response.json()["data"][-1]["detail"]) + task_ids = [task["job_id"] for task in tasks_dict] + await poll_jobs(task_ids, logs=logs, async_client=async_client) + + # Sanity-check the TSV + s3_client = get_s3_client() + expected_prefix = f"{dataset}/{version}/vector/epsg-4326" + s3_client.download_file( + DATA_LAKE_BUCKET, + f"{expected_prefix}/{dataset}_{version}_1x1.tsv", + "/tmp/1x1.tsv" + ) + with open("/tmp/1x1.tsv", "r") as f: + reader = csv.reader(f, dialect=csv.excel_tab) + header = next(reader) + assert header == ["gfw_geostore_id", "tile_id", "tcl", "glad", "geom"] + + assert "60N_010E" in next(reader)