Skip to content

Commit

Permalink
Merge pull request #156 from SnowEx/campaign_observations
Browse files Browse the repository at this point in the history 
Campaign observations - Abstract Point and Image data metadata
  • Loading branch information
@jomey
jomey authored Oct 22, 2024
2 parents b4901ab + e87c564 commit ec449ee
Show file tree
Hide file tree
Showing 41 changed files with 1,422 additions and 420 deletions.
2 changes: 2 additions & 0 deletions pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ dependencies = [

[project.optional-dependencies]
dev = [
"factory_boy<4",
"pytest-factoryboy<3",
"pytest",
"pytest-cov",
"sphinx-autobuild<=2024.5",
Expand Down
293 changes: 193 additions & 100 deletions snowexsql/api.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import logging
import os
from contextlib import contextmanager

import geoalchemy2.functions as gfunc
Expand All @@ -10,9 +11,8 @@

from snowexsql.conversions import query_to_geopandas, raster_to_rasterio
from snowexsql.db import get_db
from snowexsql.tables import ImageData, LayerData, PointData, Instrument, \
Observer, Site, Campaign, MeasurementType, DOI

from snowexsql.tables import Campaign, DOI, ImageData, Instrument, LayerData, \
MeasurementType, Observer, PointData, PointObservation, Site

LOG = logging.getLogger(__name__)
DB_NAME = 'snow:[email protected]/snowex'
Expand Down Expand Up @@ -102,6 +102,22 @@ def _filter_observers(cls, qry, v):
).filter(Observer.name == v)
return qry

@classmethod
def _filter_instrument(cls, qry, value):
return qry.filter(
cls.MODEL.instrument.has(name=value)
)

@classmethod
def _filter_measurement_type(cls, qry, value):
return qry.join(
cls.MODEL.measurement_type
).filter(MeasurementType.name == value)

@classmethod
def _filter_doi(cls, qry, value):
return qry.join(cls.MODEL.doi).filter(DOI.doi == value)

@classmethod
def extend_qry(cls, qry, check_size=True, **kwargs):
if cls.MODEL is None:
Expand All @@ -111,12 +127,15 @@ def extend_qry(cls, qry, check_size=True, **kwargs):
for k, v in kwargs.items():
# Handle special operations
if k in cls.ALLOWED_QRY_KWARGS:

qry_model = cls.MODEL
# Logic for filtering on date with LayerData
if "date" in k and cls.MODEL == LayerData:
qry = qry.join(LayerData.site)
qry_model = Site
else:
qry_model = cls.MODEL
elif cls.MODEL == PointData:
qry = qry.join(PointData.observation)

# standard filtering using qry.filter
if isinstance(v, list):
filter_col = getattr(qry_model, k)
Expand All @@ -137,17 +156,17 @@ def extend_qry(cls, qry, check_size=True, **kwargs):
# Filter boundary
if "_greater_equal" in k:
key = k.split("_greater_equal")[0]
filter_col = getattr(qry_model, key)
qry = qry.filter(filter_col >= v)
qry = qry.filter(
getattr(qry_model, key) >= v
)
elif "_less_equal" in k:
key = k.split("_less_equal")[0]
filter_col = getattr(qry_model, key)
qry = qry.filter(filter_col <= v)
# Filter linked columns
elif k == "instrument":
qry = qry.filter(
qry_model.instrument.has(name=v)
getattr(qry_model, key) <= v
)
# Filter linked columns
elif k == "instrument":
qry = cls._filter_instrument(qry, v)
elif k == "campaign":
qry = cls._filter_campaign(qry, v)
elif k == "site_id":
Expand All @@ -157,17 +176,14 @@ def extend_qry(cls, qry, check_size=True, **kwargs):
elif k == "observer":
qry = cls._filter_observers(qry, v)
elif k == "doi":
qry = qry.join(
qry_model.doi
).filter(DOI.doi == v)
qry = cls._filter_doi(qry, v)
elif k == "type":
qry = qry.join(
qry_model.measurement
).filter(MeasurementType.name == v)
qry = cls._filter_measurement_type(qry, v)
# Filter to exact value
else:
filter_col = getattr(qry_model, k)
qry = qry.filter(filter_col == v)
qry = qry.filter(
getattr(qry_model, k) == v
)
LOG.debug(
f"Filtering {k} to list {v}"
)
Expand Down Expand Up @@ -207,6 +223,103 @@ def from_unique_entries(cls, columns_to_search, **kwargs):

return results

@classmethod
def from_filter(cls, **kwargs):
"""
Get data for the class by filtering by allowed arguments. The allowed
filters are cls.ALLOWED_QRY_KWARGS.
"""
with db_session(cls.DB_NAME) as (session, engine):
try:
qry = session.query(cls.MODEL)
qry = cls.extend_qry(qry, **kwargs)

# For debugging in the test suite and not recommended
# in production
# https://docs.sqlalchemy.org/en/20/faq/sqlexpressions.html#rendering-postcompile-parameters-as-bound-parameters ## noqa
if 'DEBUG_QUERY' in os.environ:
full_sql_query = qry.statement.compile(
compile_kwargs={"literal_binds": True}
)
print("\n ** SQL query **")
print(full_sql_query)

df = query_to_geopandas(qry, engine)
except Exception as e:
session.close()
LOG.error("Failed query for PointData")
raise e

return df

@classmethod
def from_area(cls, shp=None, pt=None, buffer=None, crs=26912, **kwargs):
"""
Get data for the class within a specific shapefile or
within a point and a known buffer
Args:
shp: shapely geometry in which to filter
pt: shapely point that will have a buffer applied in order
to find search area
buffer: in same units as point
crs: integer crs to use
kwargs: for more filtering or limiting (cls.ALLOWED_QRY_KWARGS)
Returns: Geopandas dataframe of results
"""
if shp is None and pt is None:
raise ValueError(
"Inputs must be a shape description or a point and buffer"
)
if (pt is not None and buffer is None) or \
(buffer is not None and pt is None):
raise ValueError("pt and buffer must be given together")
with db_session(cls.DB_NAME) as (session, engine):
try:
if shp is not None:
qry = session.query(cls.MODEL)
# Filter geometry based on Site for LayerData
if cls.MODEL == LayerData:
qry = qry.join(cls.MODEL.site).filter(
func.ST_Within(
Site.geom, from_shape(shp, srid=crs)
)
)
else:
qry = qry.filter(
func.ST_Within(
cls.MODEL.geom, from_shape(shp, srid=crs)
)
)
qry = cls.extend_qry(qry, check_size=True, **kwargs)
df = query_to_geopandas(qry, engine)
else:
qry_pt = from_shape(pt)
qry = session.query(
gfunc.ST_SetSRID(
func.ST_Buffer(qry_pt, buffer), crs
)
)

buffered_pt = qry.all()[0][0]
qry = session.query(cls.MODEL)
# Filter geometry based on Site for LayerData
if cls.MODEL == LayerData:
qry = qry.join(cls.MODEL.site).filter(
func.ST_Within(Site.geom, buffered_pt)
)
else:
qry = qry.filter(
func.ST_Within(cls.MODEL.geom, buffered_pt)
)
qry = cls.extend_qry(qry, check_size=True, **kwargs)
df = query_to_geopandas(qry, engine)
except Exception as e:
session.close()
raise e

return df

@property
def all_site_names(self):
"""
Expand Down Expand Up @@ -287,98 +400,78 @@ class PointMeasurements(BaseDataset):
MODEL = PointData

@classmethod
def from_filter(cls, **kwargs):
"""
Get data for the class by filtering by allowed arguments. The allowed
filters are cls.ALLOWED_QRY_KWARGS.
"""
with db_session(cls.DB_NAME) as (session, engine):
try:
qry = session.query(cls.MODEL)
qry = cls.extend_qry(qry, **kwargs)
df = query_to_geopandas(qry, engine)
except Exception as e:
session.close()
LOG.error("Failed query for PointData")
raise e
def _filter_campaign(cls, qry, value):
return qry.join(
cls.MODEL.observation
).join(
PointObservation.campaign
).filter(
Campaign.name == value
)

return df
@classmethod
def _filter_measurement_type(cls, qry, value):
return qry.join(
cls.MODEL.observation
).join(
PointObservation.measurement_type
).filter(
MeasurementType.name == value
)

@classmethod
def from_area(cls, shp=None, pt=None, buffer=None, crs=26912, **kwargs):
"""
Get data for the class within a specific shapefile or
within a point and a known buffer
Args:
shp: shapely geometry in which to filter
pt: shapely point that will have a buffer applied in order
to find search area
buffer: in same units as point
crs: integer crs to use
kwargs: for more filtering or limiting (cls.ALLOWED_QRY_KWARGS)
Returns: Geopandas dataframe of results
def _filter_instrument(cls, qry, value):
return qry.join(
cls.MODEL.observation
).join(
PointObservation.instrument
).filter(
Instrument.name == value
)

"""
if shp is None and pt is None:
raise ValueError(
"Inputs must be a shape description or a point and buffer"
)
if (pt is not None and buffer is None) or \
(buffer is not None and pt is None):
raise ValueError("pt and buffer must be given together")
with db_session(cls.DB_NAME) as (session, engine):
try:
if shp is not None:
qry = session.query(cls.MODEL)
# Filter geometry based on Site for LayerData
if cls.MODEL == LayerData:
qry = qry.join(cls.MODEL.site).filter(
func.ST_Within(
Site.geom, from_shape(shp, srid=crs)
)
)
else:
qry = qry.filter(
func.ST_Within(
cls.MODEL.geom, from_shape(shp, srid=crs)
)
)
qry = cls.extend_qry(qry, check_size=True, **kwargs)
df = query_to_geopandas(qry, engine)
else:
qry_pt = from_shape(pt)
qry = session.query(
gfunc.ST_SetSRID(
func.ST_Buffer(qry_pt, buffer), crs
)
)
@classmethod
def _filter_doi(cls, qry, value):
return qry.join(
cls.MODEL.observation
).join(
PointObservation.doi
).filter(
DOI.doi == value
)

buffered_pt = qry.all()[0][0]
qry = session.query(cls.MODEL)
# Filter geometry based on Site for LayerData
if cls.MODEL == LayerData:
qry = qry.join(cls.MODEL.site).filter(
func.ST_Within(Site.geom, buffered_pt)
)
else:
qry = qry.filter(
func.ST_Within(cls.MODEL.geom, buffered_pt)
)
qry = cls.extend_qry(qry, check_size=True, **kwargs)
df = query_to_geopandas(qry, engine)
except Exception as e:
session.close()
raise e
@classmethod
def _filter_observers(cls, qry, value):
return qry.join(
cls.MODEL.observation
).join(
PointObservation.observer
).filter(
Observer.name == value
)

return df
@property
def all_instruments(self):
"""
Return all distinct instruments in the data
"""
with db_session(self.DB_NAME) as (session, engine):
result = session.query(Instrument.name).filter(
Instrument.id.in_(
session.query(PointObservation.instrument_id).distinct()
)
).all()
return self.retrieve_single_value_result(result)


class TooManyRastersException(Exception):
""" Exceptiont to report to users that their query will produce too many rasters"""
"""
Exception to report to users that their query will produce too many
rasters
"""
pass


class LayerMeasurements(PointMeasurements):
class LayerMeasurements(BaseDataset):
"""
API class for access to LayerData
"""
Expand Down
7 changes: 4 additions & 3 deletions snowexsql/conversions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
filetypes, datatypes, etc. Many tools here will be useful for most end users
of the database.
"""

import geopandas as gpd
import pandas as pd
from geoalchemy2.shape import to_shape
Expand Down Expand Up @@ -54,8 +53,10 @@ def query_to_geopandas(query, engine, **kwargs):
# Fill out the variables in the query
sql = query.statement.compile(dialect=postgresql.dialect())

# Get dataframe from geopandas using the query and engine
df = gpd.GeoDataFrame.from_postgis(sql, engine, **kwargs)
# Get dataframe from geopandas using the query and the DB connection.
# By passing in the actual connection, we maintain ownership of it and
# keep it alive until we close it.
df = gpd.read_postgis(sql, engine.connect(), **kwargs)

return df

Expand Down
Loading

0 comments on commit ec449ee

Please sign in to comment.