GTC-2618 Allow appending layers to existing tables

app/models/pydantic/authentication.py

@@ -16,7 +16,7 @@ class SignUpRequestIn(StrictBaseModel):
 
 class User(BaseModel):
     id: str
-    name: str
+    name: Optional[str]
     email: EmailStr
     createdAt: datetime
     role: str

app/models/pydantic/creation_options.py

@@ -226,10 +226,8 @@ class VectorSourceCreationOptions(StrictBaseModel):
     def validate_source_uri(cls, v, values, **kwargs):
         if values.get("source_driver") == VectorDrivers.csv:
             assert len(v) >= 1, "CSV sources require at least one input file"
-        else:
-            assert (
-                len(v) == 1
-            ), "Non-CSV vector sources require one and only one input file"
+        elif values.get("source_driver") in [VectorDrivers.esrijson, VectorDrivers.shp, VectorDrivers.geojson_seq, VectorDrivers.geojson]:
+            assert (len(v) == 1), "GeoJSON and ESRI Shapefile vector sources require one and only one input file"
         return v
 
 

app/models/pydantic/versions.py

@@ -1,7 +1,8 @@
 from typing import List, Optional, Tuple, Union
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, validator
 
+from ..enum.creation_options import VectorDrivers
 from ..enum.versions import VersionStatus
 from .base import BaseRecord, StrictBaseModel
 from .creation_options import SourceCreationOptions
@@ -59,7 +60,14 @@ class VersionUpdateIn(StrictBaseModel):
 
 class VersionAppendIn(StrictBaseModel):
     source_uri: List[str]
-
+    source_driver: Optional[VectorDrivers] = Field(
+        None, description="Driver of source file. Must be an OGR driver"
+    )
+    layers: Optional[List[str]] = Field(
+        None,
+        description="List of layer names to append to version. "
+        "If not set, all layers in source_uri will be appended.",
+    )
 
 class VersionResponse(Response):
     data: Version

app/routes/assets/asset.py

@@ -25,12 +25,12 @@
 
 # from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Path, status
 from fastapi.responses import ORJSONResponse
-from starlette.responses import JSONResponse
+from starlette.responses import JSONResponse, RedirectResponse
 
 from app.models.pydantic.responses import Response
 from app.settings.globals import API_URL
+from ..datasets.downloads import _get_presigned_url
 
-from ...authentication.token import is_admin
 from ...crud import assets
 from ...crud import metadata as metadata_crud
 from ...crud import tasks
@@ -67,7 +67,7 @@
     delete_static_vector_tile_cache_assets,
 )
 from ...utils.paginate import paginate_collection
-from ...utils.path import infer_srid_from_grid
+from ...utils.path import infer_srid_from_grid, split_s3_path
 from ..assets import asset_response
 from ..tasks import paginated_tasks_response, tasks_response
 
@@ -310,6 +310,28 @@ async def get_extent(asset_id: UUID = Path(...)):
     return ExtentResponse(data=extent)
 
 
+@router.get(
+    "/{asset_id}/tiles_info",
+    response_class=RedirectResponse,
+    tags=["Assets"],
+    status_code=307,
+)
+async def get_tiles_info(asset_id: UUID = Path(...)):
+    asset: ORMAsset = await assets.get_asset(asset_id)
+
+    if asset.asset_type != AssetType.raster_tile_set:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Tiles information only available for raster tile sets"
+        )
+
+    bucket, asset_key = split_s3_path(asset.asset_uri)
+    tiles_geojson_key = asset_key.replace("{tile_id}.tif", "tiles.geojson")
+    presigned_url = await _get_presigned_url(bucket, tiles_geojson_key)
+
+    return RedirectResponse(url=presigned_url)
+
+
 @router.get(
     "/{asset_id}/stats",
     response_class=ORJSONResponse,

app/routes/datasets/versions.py

@@ -13,6 +13,7 @@
 from copy import deepcopy
 from typing import Any, Dict, List, Optional, Sequence, Tuple, Union, cast
 from urllib.parse import urlparse
+import fiona
 
 from fastapi import (
     APIRouter,
@@ -211,8 +212,7 @@ async def update_version(
     "/{dataset}/{version}/append",
     response_class=ORJSONResponse,
     tags=["Versions"],
-    response_model=VersionResponse,
-    deprecated=True,
+    response_model=VersionResponse
 )
 async def append_to_version(
     *,
@@ -240,13 +240,36 @@ async def append_to_version(
     # For the background task, we only need the new source uri from the request
     input_data = {"creation_options": deepcopy(default_asset.creation_options)}
     input_data["creation_options"]["source_uri"] = request.source_uri
+
+    # If source_driver is "text", this is a datapump request
+    if input_data["creation_options"]["source_driver"] != "text":
+        # Verify that source_driver is not None
+        if input_data["creation_options"]["source_driver"] is None:
+            raise HTTPException(
+                status_code=400,
+                detail="Source driver must be specified for non-datapump requests."
+            )
+
+        # Append the new layers to the existing ones
+        if input_data["creation_options"].get("layers") is None: # ERROR: layers is not defined
+            input_data["creation_options"]["layers"] = request.layers
+        elif request.layers is not None:
+            input_data["creation_options"]["layers"] += request.layers
+        else:
+            input_data["creation_options"]["layers"] = request.layers
+
     background_tasks.add_task(
         append_default_asset, dataset, version, input_data, default_asset.asset_id
     )
 
     # We now want to append the new uris to the existing ones and update the asset
     update_data = {"creation_options": deepcopy(default_asset.creation_options)}
-    update_data["creation_options"]["source_uri"] += request.source_uri
+    update_data["creation_options"]["source_uri"] += request.source_uri # ERROR: only one source_uri is allowed
+    if input_data["creation_options"].get("layers") is not None:
+        if update_data["creation_options"]["layers"] is not None:
+            update_data["creation_options"]["layers"] += request.layers
+        else:
+            update_data["creation_options"]["layers"] = request.layers
     await assets.update_asset(default_asset.asset_id, **update_data)
 
     version_orm: ORMVersion = await versions.get_version(dataset, version)
@@ -536,6 +559,17 @@ async def _version_response(
 
     return VersionResponse(data=Version(**data))
 
+#def _verify_layer_exists(source_uri: List[str], layes: List[str]) -> None:
+#    with fiona.open(source_uri[0].replace("s3://", "/vsizip//vsis3/"), "r") as src:
+#        layers = src.layer_names
+#        for layer in layers:
+#            if layer in layers:
+#                return
+#            else:
+#                raise HTTPException(
+#                    status_code=400,
+#                    detail=f"Layer {layer} not found in source file."
+#                )
 
 def _verify_source_file_access(sources: List[str]) -> None:
 

terraform/vars/terraform-production.tfvars

@@ -5,6 +5,8 @@ rw_api_url                = "https://api.resourcewatch.org"
 desired_count             = 2
 auto_scaling_min_capacity = 2
 auto_scaling_max_capacity = 15
+fargate_cpu               = 2048
+fargate_memory            = 4096
 lambda_analysis_workspace = "default"
 key_pair                  = "dmannarino_gfw"
 new_relic_license_key_arn = "arn:aws:secretsmanager:us-east-1:401951483516:secret:newrelic/license_key-CyqUPX"

tests/__init__.py

@@ -49,6 +49,9 @@
 SHP_NAME = "test.shp.zip"
 SHP_PATH = os.path.join(os.path.dirname(__file__), "fixtures", SHP_NAME)
 
+GPKG_NAME = "test.gpkg.zip"
+GPKG_PATH = os.path.join(os.path.dirname(__file__), "fixtures", GPKG_NAME)
+
 BUCKET = "test-bucket"
 PORT = 9000
 

tests/conftest.py

@@ -48,6 +48,8 @@
     GEOJSON_NAME2,
     GEOJSON_PATH,
     GEOJSON_PATH2,
+    GPKG_NAME,
+    GPKG_PATH,
     PORT,
     SHP_NAME,
     SHP_PATH,
@@ -308,6 +310,7 @@ def copy_fixtures():
     s3_client.upload_file(CSV2_PATH, BUCKET, CSV2_NAME)
     s3_client.upload_file(TSV_PATH, BUCKET, TSV_NAME)
     s3_client.upload_file(SHP_PATH, BUCKET, SHP_NAME)
+    s3_client.upload_file(GPKG_PATH, BUCKET, GPKG_NAME)
     s3_client.upload_file(APPEND_TSV_PATH, BUCKET, APPEND_TSV_NAME)
 
     # upload a separate for each row so we can test running large numbers of sources in parallel

tests/routes/datasets/test_versions.py

@@ -7,7 +7,7 @@
 
 from app.settings.globals import S3_ENTRYPOINT_URL
 from app.utils.aws import get_s3_client
-from tests import BUCKET, DATA_LAKE_BUCKET, SHP_NAME
+from tests import BUCKET, DATA_LAKE_BUCKET, SHP_NAME, GPKG_NAME
 from tests.conftest import FAKE_INT_DATA_PARAMS
 from tests.tasks import MockCloudfrontClient
 from tests.utils import (
@@ -326,7 +326,8 @@ async def test_invalid_source_uri(async_client: AsyncClient):
 
     # Test appending to a version that exists
     response = await async_client.post(
-        f"/dataset/{dataset}/{version}/append", json={"source_uri": source_uri}
+        f"/dataset/{dataset}/{version}/append", 
+        json={"source_uri": source_uri, "source_driver": "ESRI Shapefile"}
     )
     assert response.status_code == 400
     assert response.json()["status"] == "failed"