Merge branch 'develop' into shared-dev-lb

.github/workflows/terraform_destroy_on_delete.yaml

@@ -4,7 +4,7 @@ on: [delete]
 
 jobs:
   build:
-    if: contains(github.event.ref_type, 'branch') &&  (! contains(github.event.ref, 'master')) && (! contains(github.event.ref, 'develop'))
+    if: contains(github.event.ref_type, 'branch') &&  (! github.event.ref == 'master') && (! github.event.ref == 'develop')
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v1

.github/workflows/terraform_plan.yaml

@@ -9,15 +9,16 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - name: Plan production
-      if: success() && contains(github.base_ref, 'master')
+      if: success() && github.base_ref == 'master'
       env:
         ENV: production
         AWS_ACCESS_KEY_ID: ${{ secrets.aws_key_production }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.aws_secret_production }}
         AWS_REGION: ${{ secrets.aws_region_production }}
       run: ./scripts/infra plan -w ${{ github.base_ref }}
+
     - name: Plan staging
-      if: success() && contains(github.base_ref, 'develop')
+      if: success() && github.base_ref == 'develop'
       env:
         ENV: staging
         AWS_ACCESS_KEY_ID: ${{ secrets.aws_key_staging }}

.isort.cfg

@@ -2,4 +2,4 @@
 line_length = 88
 multi_line_output = 3
 include_trailing_comma = True
-known_third_party = _pytest,aenum,affine,aiohttp,alembic,async_lru,asyncpg,aws_utils,boto3,botocore,click,docker,errors,fastapi,fiona,gdal_utils,geoalchemy2,geojson,gfw_pixetl,gino,gino_starlette,google,httpx,httpx_auth,logger,logging_utils,moto,numpy,orjson,osgeo,pandas,pendulum,pglast,psutil,psycopg2,pydantic,pyproj,pytest,pytest_asyncio,rasterio,retrying,shapely,sqlalchemy,sqlalchemy_utils,starlette,tileputty,typer
+known_third_party = _pytest,aenum,affine,aiohttp,alembic,async_lru,asyncpg,aws_utils,boto3,botocore,click,docker,errors,fastapi,fiona,gdal_utils,geoalchemy2,geojson,gfw_pixetl,gino,gino_starlette,google,httpx,httpx_auth,logger,logging_utils,moto,numpy,orjson,osgeo,pandas,pendulum,pglast,psutil,psycopg2,pydantic,pyproj,pytest,pytest_asyncio,rasterio,shapely,sqlalchemy,sqlalchemy_utils,starlette,tileputty,typer

app/crud/metadata.py

@@ -181,12 +181,44 @@ async def get_asset_metadata(asset_id: UUID):
 async def update_asset_metadata(asset_id: UUID, **data) -> ORMAssetMetadata:
     """Update asset metadata."""
     fields = data.pop("fields", None)
+    bands = data.pop("bands", None)
 
     asset_metadata: ORMAssetMetadata = await get_asset_metadata(asset_id)
 
     if data:
         await asset_metadata.update(**data).apply()
 
+    bands_metadata = []
+    if bands:
+        for band in bands:
+            try:
+                pixel_meaning = band.pop("pixel_meaning")
+                band_metadata = await update_band_metadata(
+                    asset_metadata.id, pixel_meaning, **band
+                )
+            except RecordNotFoundError:
+                bands_metadata = await create_raster_band_metadata(
+                    asset_metadata.id, **bands
+                )
+            bands_metadata.append(band_metadata)
+
+        asset_metadata.bands = bands_metadata
+
+    fields_metadata = []
+    if fields:
+        for field in fields:
+            try:
+                field_metadata = await update_field_metadata(
+                    asset_metadata.id, field["name"], **field
+                )
+            except RecordNotFoundError:
+                field_metadata = await create_field_metadata(asset_metadata.id, **field)
+            fields_metadata.append(field_metadata)
+
+        asset_metadata.fields = fields_metadata
+
+    return asset_metadata
+
     fields_metadata = []
     if fields:
         for field in fields:
@@ -240,6 +272,18 @@ async def update_field_metadata(
     return field_metadata
 
 
+async def update_band_metadata(
+    metadata_id: UUID, pixel_meaning: str, **data
+) -> ORMFieldMetadata:
+    band_metadata: ORMRasterBandMetadata = await get_asset_raster_band(
+        metadata_id, pixel_meaning
+    )
+
+    await band_metadata.update(**data).apply()
+
+    return band_metadata
+
+
 async def get_asset_fields(asset_metadata_id: UUID) -> List[ORMFieldMetadata]:
     fields_metadata: List[ORMFieldMetadata] = await (
         ORMFieldMetadata.query.where(
@@ -272,6 +316,21 @@ async def get_asset_field(asset_metadata_id: UUID, field_name: str) -> ORMFieldM
     return field_metadata
 
 
+async def get_asset_raster_band(
+    asset_metadata_id: UUID, pixel_meaning: str
+) -> ORMRasterBandMetadata:
+    band_metadata: ORMRasterBandMetadata = await ORMRasterBandMetadata.get(
+        [asset_metadata_id, pixel_meaning]
+    )
+
+    if band_metadata is None:
+        raise RecordNotFoundError(
+            f"No band metadata record found for pixel meaning {pixel_meaning}."
+        )
+
+    return band_metadata
+
+
 def update_metadata(row: Base, parent: Base):
     """Dynamically update metadata with parent metadata.
 

app/models/orm/asset_metadata.py

@@ -52,8 +52,8 @@ class RasterBandMetadata(db.Model):
             name="asset_metadata_id_fk",
             onupdate="CASCADE",
             ondelete="CASCADE",
-            primary_key=True,
         ),
+        primary_key=True,
     )
     pixel_meaning = db.Column(db.String, primary_key=True)
     description = db.Column(db.String)

app/models/pydantic/versions.py

@@ -17,7 +17,8 @@ class Version(BaseRecord):
     metadata: Union[VersionMetadataOut, BaseModel]
     status: VersionStatus = VersionStatus.pending
 
-    assets: List[Tuple[str, str]] = list()
+    # Each element of assets is a tuple (asset_type, assert_uri, asset_id)
+    assets: List[Tuple[str, str, str]] = list()
 
 
 class VersionCreateIn(StrictBaseModel):

app/routes/assets/asset.py

@@ -11,11 +11,6 @@
 from typing import List, Optional, Union
 from uuid import UUID
 
-# from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Path, status
-from fastapi.responses import ORJSONResponse
-from starlette.responses import JSONResponse
-
-from app.models.pydantic.responses import Response
 from fastapi import (
     APIRouter,
     BackgroundTasks,
@@ -27,13 +22,18 @@
     status,
 )
 
+# from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Path, status
+from fastapi.responses import ORJSONResponse
+from starlette.responses import JSONResponse
+
+from app.models.pydantic.responses import Response
 from app.settings.globals import API_URL
 
 from ...authentication.token import is_admin
 from ...crud import assets
 from ...crud import metadata as metadata_crud
 from ...crud import tasks
-from ...errors import RecordAlreadyExistsError, RecordNotFoundError
+from ...errors import BadRequestError, RecordAlreadyExistsError, RecordNotFoundError
 from ...models.enum.assets import is_database_asset, is_single_file_asset
 from ...models.orm.asset_metadata import FieldMetadata as ORMFieldMetadata
 from ...models.orm.assets import Asset as ORMAsset
@@ -112,6 +112,8 @@ async def update_asset(
         row: ORMAsset = await assets.update_asset(asset_id, **input_data)
     except RecordNotFoundError as e:
         raise HTTPException(status_code=404, detail=str(e))
+    except BadRequestError as e:
+        raise HTTPException(status_code=400, detail=str(e))
     except NotImplementedError as e:
         raise HTTPException(status_code=501, detail=str(e))
 

app/routes/datasets/queries.py

@@ -7,25 +7,23 @@
 from uuid import UUID, uuid4
 
 import httpx
+from async_lru import alru_cache
 from asyncpg import DataError, InsufficientPrivilegeError, SyntaxOrAccessError
 from fastapi import APIRouter, Depends, HTTPException, Query
 from fastapi import Request as FastApiRequest
 from fastapi import Response as FastApiResponse
 from fastapi.encoders import jsonable_encoder
 from fastapi.logger import logger
-
 # from fastapi.openapi.models import APIKey
 from fastapi.responses import RedirectResponse
 from pglast import printers  # noqa
 from pglast import Node, parse_sql
 from pglast.parser import ParseError
 from pglast.printer import RawStream
 from pydantic.tools import parse_obj_as
-from sqlalchemy.sql import and_
 
 from ...authentication.token import is_gfwpro_admin_for_query
 from ...application import db
-
 # from ...authentication.api_keys import get_api_key
 from ...crud import assets
 from ...models.enum.assets import AssetType
@@ -62,7 +60,6 @@
 from ...models.enum.queries import QueryFormat, QueryType
 from ...models.orm.assets import Asset as AssetORM
 from ...models.orm.queries.raster_assets import latest_raster_tile_sets
-from ...models.orm.versions import Version as VersionORM
 from ...models.pydantic.asset_metadata import RasterTable, RasterTableRow
 from ...models.pydantic.creation_options import NoDataType
 from ...models.pydantic.geostore import Geometry, GeostoreCommon
@@ -659,13 +656,14 @@ async def _query_raster_lambda(
 
 
 def _get_area_density_name(nm):
-    """Return empty string if nm doesn't not have an area-density suffix, else
+    """Return empty string if nm doesn't have an area-density suffix, else
     return nm with the area-density suffix removed."""
     for suffix in AREA_DENSITY_RASTER_SUFFIXES:
         if nm.endswith(suffix):
             return nm[:-len(suffix)]
     return ""
 
+
 def _get_default_layer(dataset, pixel_meaning):
     default_type = pixel_meaning
     area_density_name = _get_area_density_name(default_type)
@@ -683,6 +681,7 @@ def _get_default_layer(dataset, pixel_meaning):
         return f"{dataset}__{default_type}"
 
 
+@alru_cache(maxsize=16, ttl=300.0)
 async def _get_data_environment(grid: Grid) -> DataEnvironment:
     # get all raster tile set assets with the same grid.
     latest_tile_sets = await db.all(latest_raster_tile_sets, {"grid": grid})

app/routes/datasets/versions.py

@@ -500,14 +500,14 @@ async def _version_response(
     associated assets."""
 
     assets: List[ORMAsset] = (
-        await ORMAsset.select("asset_type", "asset_uri")
+        await ORMAsset.select("asset_type", "asset_uri", "asset_id")
         .where(ORMAsset.dataset == dataset)
         .where(ORMAsset.version == version)
         .where(ORMAsset.status == AssetStatus.saved)
         .gino.all()
     )
     data = Version.from_orm(data).dict(by_alias=True)
-    data["assets"] = [(asset[0], asset[1]) for asset in assets]
+    data["assets"] = [(asset[0], asset[1], str(asset[2])) for asset in assets]
 
     return VersionResponse(data=Version(**data))
 
@@ -560,7 +560,7 @@ def _verify_source_file_access(sources: List[str]) -> None:
         raise HTTPException(
             status_code=400,
             detail=(
-                "Cannot access all of the source files. "
+                "Cannot access all of the source files (non-existent or access denied). "
                 f"Invalid sources: {invalid_sources}"
             ),
         )

app/tasks/delete_assets.py

@@ -11,7 +11,7 @@
 async def delete_all_assets(dataset: str, version: str) -> None:
     await delete_database_table_asset(dataset, version)
     delete_s3_objects(DATA_LAKE_BUCKET, f"{dataset}/{version}/")
-    # expire_s3_objects(TILE_CACHE_BUCKET, f"{dataset}/{version}/")
+    expire_s3_objects(TILE_CACHE_BUCKET, f"{dataset}/{version}/")
     flush_cloudfront_cache(TILE_CACHE_CLOUDFRONT_ID, [f"/{dataset}/{version}/*"])
 
 

app/utils/aws.py

@@ -1,8 +1,10 @@
 from typing import Any, Dict, List, Optional, Sequence
 
 import boto3
+import botocore
 import httpx
 from httpx_auth import AWS4Auth
+from fastapi.logger import logger
 
 from ..settings.globals import (
     AWS_REGION,
@@ -108,7 +110,11 @@ def get_aws_files(
             if exit_after_max and num_matches >= exit_after_max:
                 break
 
-    except s3_client.exceptions.NoSuchBucket:
+    # Strangely, s3_client.exceptions has NoSuchBucket, but doesn't have
+    # AccessDenied, even though you can get that error, so we just catch all botocore
+    # exceptions.
+    except botocore.exceptions.ClientError as error:
+        logger.warning(f"get_aws_file: {error}")
         matches = list()
 
     return matches

batch/scripts/create_vector_tile_cache.sh

@@ -34,7 +34,7 @@ echo "Fetch NDJSON data from Data Lake ${SRC} -> ${DATASET}"
 aws s3 cp "${SRC}" "${DATASET}" --no-progress
 
 echo "Build Tile Cache"
-tippecanoe -Z"${MIN_ZOOM}" -z"${MAX_ZOOM}" -e tilecache "${STRATEGY[@]}" -P -n "${DATASET}" "${DATASET}"
+tippecanoe -Z"${MIN_ZOOM}" -z"${MAX_ZOOM}" -e tilecache "${STRATEGY[@]}" -P -n "${DATASET}" "${DATASET}" --preserve-input-order
 
 echo "Upload tiles to S3"
 tileputty tilecache --bucket "${TILE_CACHE}" --dataset "${DATASET}" --version "${VERSION}" --implementation "${IMPLEMENTATION}" --cores "${NUM_PROCESSES}"

newrelic.ini

@@ -37,7 +37,7 @@ app_name = GFW Data API
 
 # New Relic offers distributed tracing for monitoring and analyzing modern
 # distributed systems.Enable distributed tracing.
-distributed_tracing.enabled = <useDistributedTracing>
+# distributed_tracing.enabled = false
 
 # When "true", the agent collects performance data about your
 # application and reports this data to the New Relic UI at
@@ -115,7 +115,7 @@ high_security = false
 # then add "request.parameters.*" to the "attributes.include"
 # setting. For details about attributes configuration, please
 # consult the documentation.
-# attributes.include = request.parameters.*
+attributes.include = request.parameters.*
 
 # The transaction tracer captures deep information about slow
 # transactions and sends this to the UI on a periodic basis. The
@@ -203,8 +203,10 @@ monitor_mode = false
 [newrelic:staging]
 app_name = GFW Data API (Staging)
 monitor_mode = true
+distributed_tracing.enabled = false
 
 [newrelic:production]
 monitor_mode = true
+distributed_tracing.enabled = true
 
 # ---------------------------------------------------------------------------