diff --git a/cosmo/monitors/dark_data_models.py b/cosmo/monitors/dark_data_models.py new file mode 100644 index 0000000..4edae6d --- /dev/null +++ b/cosmo/monitors/dark_data_models.py @@ -0,0 +1,51 @@ +import pandas as pd +import os +from glob import glob +from monitorframe.datamodel import BaseDataModel + +from ..filesystem import find_files, data_from_exposures +from .. import SETTINGS + +FILES_SOURCE = SETTINGS['filesystem']['source'] + + +class DarkDataModel(BaseDataModel): + cosmo_layout = False + program_id = ['15771', '15533/', '14940/', '14520/', '14436/', '13968/', '13521/', '13121/', '12716/', '12423/', + '11895/'] + # subdir_pattern = '?????' + + def get_new_data(self): + header_request = { + 0: ['ROOTNAME', 'SEGMENT'], + 1: ['EXPTIME', 'EXPSTART'] + } + + table_request = { + 1: ['PHA', 'XCORR', 'YCORR', 'TIME'], + 3: ['TIME', 'LATITUDE', 'LONGITUDE'] + } + + files = [] + + for prog_id in self.program_id: + + new_files_source = os.path.join(FILES_SOURCE, prog_id) + files += find_files('*corrtag*', data_dir=new_files_source) + + if self.model is not None: + currently_ingested = [item.FILENAME for item in self.model.select(self.model.FILENAME)] + + for file in currently_ingested: + files.remove(file) + + if not files: # No new files + return pd.DataFrame() + + data_results = data_from_exposures( + files, + header_request=header_request, + table_request=table_request + ) + + return data_results diff --git a/cosmo/monitors/dark_monitors.py b/cosmo/monitors/dark_monitors.py new file mode 100644 index 0000000..4d01a10 --- /dev/null +++ b/cosmo/monitors/dark_monitors.py @@ -0,0 +1,156 @@ +from typing import Any + +import numpy as np +import plotly.graph_objs as go +import pandas as pd + +from itertools import repeat + +from monitorframe.monitor import BaseMonitor +from .data_models import FUVDarkDataModel +from ..monitor_helpers import explode_df, absolute_time +from .. import SETTINGS + +COS_MONITORING = SETTINGS['output'] + + +def dark_filter(df_row, filter_pha, location): + good_pha = (2, 23) + time_step = 25 + time_bins = df_row['TIME_3'][::time_step] + lat = df_row['LATITUDE'][::time_step][:-1] + lon = df_row['LONGITUDE'][::time_step][:-1] + event_df = df_row[['SEGMENT', 'XCORR', 'YCORR', 'PHA', 'TIME']].to_frame().T + event_df = explode_df(event_df, ['XCORR', 'YCORR', 'PHA', 'TIME']) + npix = (location[1] - location[0]) * (location[3] - location[2]) + index = np.where((event_df['XCORR'] > location[0]) & + (event_df['XCORR'] < location[1]) & + (event_df['YCORR'] > location[2]) & + (event_df['YCORR'] < location[3])) + filtered_row = event_df.iloc[index].reset_index(drop=True) + + if filter_pha: + filtered_row = filtered_row[(filtered_row['PHA'] > good_pha[0]) & (filtered_row['PHA'] < good_pha[1])] + + counts = np.histogram(filtered_row.TIME, bins=time_bins)[0] + + date = absolute_time( + expstart=list(repeat(df_row['EXPSTART'], len(time_bins))), time=time_bins.tolist() + ).to_datetime()[:-1] + + dark_rate = counts / npix / time_step + + return pd.DataFrame({'segment': df_row['SEGMENT'], 'darks': [dark_rate], 'date': [date], + 'ROOTNAME': df_row['ROOTNAME']}) + + +class FUVDarkMonitor(BaseMonitor): + """Abstracted FUV Dark Monitor. Not meant to be used directly but rather inherited by specific segment and region + dark monitors""" + labels = ['ROOTNAME'] + output = COS_MONITORING + docs = "https://spacetelescope.github.io/cosmo/monitors.html#fuv-dark-rate-monitors" + segment = None + location = None + plottype = 'scatter' + x = 'date' + y = 'darks' + + def get_data(self) -> Any: + filtered_rows = [] + for _, row in self.model.new_data.iterrows(): + if row.EXPSTART == 0: + continue + if row.SEGMENT == self.segment: + filtered_rows.append(dark_filter(row, True, self.location)) + filtered_df = pd.concat(filtered_rows).reset_index(drop=True) + + return explode_df(filtered_df, ['darks', 'date']) + + def store_results(self): + # TODO: Define results to store + pass + + def track(self): + # TODO: Define something to track + pass + + +class FUVABottomDarkMonitor(FUVDarkMonitor): + """FUVA dark monitor for bottom edge""" + data_model = FUVDarkDataModel + segment = 'FUVA' + location = (1060, 15250, 296, 375) + name = f'FUVA Dark Monitor - Bottom' + + +class FUVALeftDarkMonitor(FUVDarkMonitor): + """FUVA dark monitor for left edge""" + name = 'FUVA Dark Monitor - Left' + data_model = FUVDarkDataModel + segment = 'FUVA' + location = (1060, 1260, 296, 734) + + +class FUVATopDarkMonitor(FUVDarkMonitor): + """FUVA dark monitor for top edge""" + name = 'FUVA Dark Monitor - Top' + data_model = FUVDarkDataModel + segment = 'FUVA' + location = (1060, 15250, 660, 734) + + +class FUVARightDarkMonitor(FUVDarkMonitor): + """FUVA dark monitor for right edge""" + name = 'FUVA Dark Monitor - Right' + data_model = FUVDarkDataModel + segment = 'FUVA' + location = (15119, 15250, 296, 734) + + +class FUVAInnerDarkMonitor(FUVDarkMonitor): + """FUVA dark monitor for inner region""" + name = 'FUVA Dark Monitor - Inner' + data_model = FUVDarkDataModel + segment = 'FUVA' + location = (1260, 15119, 375, 660) + + +class FUVBBottomDarkMonitor(FUVDarkMonitor): + """FUVB dark monitor for bottom edge""" + name = 'FUVB Dark Monitor - Bottom' + data_model = FUVDarkDataModel + segment = 'FUVB' + location = (809, 15182, 360, 405) + + +class FUVBLeftDarkMonitor(FUVDarkMonitor): + """FUVB dark monitor for left edge""" + name = 'FUVB Dark Monitor - Left' + data_model = FUVDarkDataModel + segment = 'FUVB' + location = (809, 1000, 360, 785) + + +class FUVBTopDarkMonitor(FUVDarkMonitor): + """FUVB dark monitor for top edge""" + name = 'FUVB Dark Monitor - Top' + data_model = FUVDarkDataModel + segment = 'FUVB' + location = (809, 15182, 740, 785) + + +class FUVBRightDarkMonitor(FUVDarkMonitor): + """FUVB dark monitor for right edge""" + name = 'FUVB Dark Monitor - Right' + data_model = FUVDarkDataModel + segment = 'FUVB' + location = (14990, 15182, 360, 785) + + +class FUVBInnerDarkMonitor(FUVDarkMonitor): + """FUVB dark monitor for inner region""" + name = 'FUVB Dark Monitor - Inner' + data_model = FUVDarkDataModel + segment = 'FUVB' + location = (1000, 14990, 405, 740) diff --git a/cosmo/monitors/data_models.py b/cosmo/monitors/data_models.py index 503251e..650754c 100644 --- a/cosmo/monitors/data_models.py +++ b/cosmo/monitors/data_models.py @@ -1,5 +1,7 @@ import pandas as pd import numpy as np +import os +from glob import glob from typing import List from monitorframe.datamodel import BaseDataModel @@ -192,3 +194,50 @@ def get_new_data(self): data_results = data_results.replace([np.inf, -np.inf], np.nan).dropna().reset_index(drop=True) return data_results[~data_results.EXPTYPE.str.contains('ACQ|DARK|FLAT')] + + +class FUVDarkDataModel(BaseDataModel): + files_source = FILES_SOURCE + cosmo_layout = False + + """program_ids = ['15771/', '15533/', '14940/', '14520/', '14436/', '13968/', '13521/', '13121/', '12716/', '12423/', + '11895/']""" + program_ids = ['15771/', '15533/'] + + primary_key = 'ROOTNAME' + + def get_new_data(self): + """Retrieve data""" + header_request = { + 0: ['ROOTNAME', 'SEGMENT'], + 1: ['EXPTIME', 'EXPSTART'] + } + + table_request = { + 1: ['PHA', 'XCORR', 'YCORR', 'TIME'], + 3: ['TIME', 'LATITUDE', 'LONGITUDE'] + } + + files = [] + + for prog_id in self.program_ids: + + new_files_source = os.path.join(FILES_SOURCE, prog_id) + files += find_files('*corrtag*', data_dir=new_files_source, subdir_pattern=None) + + if self.model is not None: + currently_ingested = [item.FILENAME for item in self.model.select(self.model.FILENAME)] + + for file in currently_ingested: + files.remove(file) + + if not files: # No new files + return pd.DataFrame() + + data_results = data_from_exposures( + files, + header_request=header_request, + table_request=table_request + ) + + return data_results