Skip to content

Commit

Permalink
Revert all changes to latest branch
Browse files Browse the repository at this point in the history
  • Loading branch information
@chandra-tacc
chandra-tacc committed Oct 30, 2023
1 parent 8c7714e commit ac1b63a
Show file tree
Hide file tree
Showing 3 changed files with 122 additions and 116 deletions.
2 changes: 0 additions & 2 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,3 @@ flask==2.0.3
gunicorn==20.1.0
requests==2.27.0
pandas==1.3.5
retrying==1.3.4
Werkzeug==2.0.3
192 changes: 99 additions & 93 deletions src/app.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
from flask import Flask, jsonify, request
import logging
import os
import pandas as pd
import csv
Expand All @@ -8,24 +7,37 @@
from data_loading import *

# ----------------------------------------------------------------------------
# DATA PARAMETERS
# ENV Variables & DATA PARAMETERS
# ----------------------------------------------------------------------------
data_access_type = os.environ.get('DATA_ACCESS_TYPE')

current_folder = os.path.dirname(__file__)
DATA_PATH = os.path.join(current_folder,'data')
ASSETS_PATH = os.path.join(current_folder,'assets')

local_data_path = os.environ.get("LOCAL_DATA_PATH","")
local_data_date = os.environ.get("LOCAL_DATA_DATE","")

imaging_filepath = os.path.join(local_data_path,os.environ.get("IMAGING_FILE"))
qc_filepath = os.path.join(local_data_path,os.environ.get("QC_FILE"))
blood1_filepath = os.path.join(local_data_path,os.environ.get("BLOOD1_FILE"))
blood2_filepath = os.path.join(local_data_path,os.environ.get("BLOOD2_FILE"))
subjects1_filepath = os.path.join(local_data_path,os.environ.get("SUBJECTS1_FILE"))
subjects2_filepath = os.path.join(local_data_path,os.environ.get("SUBJECTS2_FILE"))

print(local_data_path, local_data_date, subjects2_filepath)


# ----------------------------------------------------------------------------
# LOAD ASSETS FILES
# ----------------------------------------------------------------------------
asset_files_dict = {
'screening_sites': 'screening_sites.csv',
'display_terms': 'A2CPS_display_terms.csv',
}

display_terms, display_terms_dict, display_terms_dict_multi = load_display_terms(ASSETS_PATH, asset_files_dict['display_terms'])

screening_sites = pd.read_csv(os.path.join(ASSETS_PATH,asset_files_dict['screening_sites']))
# Pointers to official files stored at github repository main branch
screening_sites_github_url = 'https://raw.githubusercontent.com/TACC/a2cps-datastore-weekly/main/src/assets/screening_sites.csv'
display_terms_github_url = 'https://raw.githubusercontent.com/TACC/a2cps-datastore-weekly/main/src/assets/A2CPS_display_terms.csv'

# load display terms and screening sites
screening_sites = pd.read_csv(screening_sites_github_url)
display_terms, display_terms_dict, display_terms_dict_multi = load_display_terms_from_github(display_terms_github_url)

# Columns used in reports [UPDATE THIS IF START TO USE MORE]
subjects_raw_cols_for_reports = ['ewcomments',
Expand Down Expand Up @@ -53,39 +65,16 @@
'main_record_id',
'sp_mricompatscr',
'ewdateterm']
# ----------------------------------------------------------------------------
# LOAD INITAL DATA FROM FILES
# ----------------------------------------------------------------------------

local_date = '2022-09-08'

local_imaging_data = {
'date': local_date,
'data': get_local_imaging_data(DATA_PATH)}

local_blood_data = {
'date': local_date,
'data': get_local_blood_data(DATA_PATH)}

subjects_raw = get_local_subjects_raw(DATA_PATH)
local_subjects_data = {
'date': local_date,
'data': process_subjects_data(subjects_raw,subjects_raw_cols_for_reports,screening_sites, display_terms_dict, display_terms_dict_multi)
}

local_data = {
'imaging': local_imaging_data,
'blood': local_imaging_data,
'subjects': local_subjects_data
}

# ----------------------------------------------------------------------------
# APIS
# ----------------------------------------------------------------------------
datetime_format = "%m/%d/%Y, %H:%M:%S"

apis_imaging_index = {}
data_state = 'empty'

api_data_index = {
'blood':'',
'imaging':'',
Expand All @@ -107,71 +96,93 @@
'consort':None,
}

# ----------------------------------------------------------------------------
# SIMPLE APIS
# ----------------------------------------------------------------------------
api_data_simple = {
'blood':None,
'imaging':None,
'subjects':None,
'raw': None
}

# ----------------------------------------------------------------------------
# APP
# ----------------------------------------------------------------------------

app = Flask(__name__)
app.debug = True
logging.basicConfig(filename='app.log',level=logging.DEBUG)

# APIS: try to load new data, if doesn't work, get most recent
@app.route("/api/apis")
def api_apis():
return jsonify(api_data_index)

@app.route("/api/tester")
def api_tester():
if local_data_path:
return jsonify(local_data_path)
else:
return jsonify('local_data_path not found')

@app.route("/api/imaging")
def api_imaging():
global datetime_format
global api_data_index
global api_data_cache

try:
if not api_data_index['imaging'] or not check_data_current(datetime.strptime(api_data_index['imaging'], datetime_format)):
api_date = datetime.now().strftime(datetime_format)
imaging_data = get_api_imaging_data(request)
if data_access_type != 'LOCAL':
data_date = datetime.now().strftime(datetime_format)
imaging_data = get_api_imaging_data(request)
else:
data_date = local_data_date
imaging_data = get_local_imaging_data(imaging_filepath, qc_filepath)

if imaging_data:
api_data_index['imaging'] = data_date
api_data_cache['imaging'] = imaging_data
api_data_index['imaging'] = api_date

return jsonify({'date': api_data_index['imaging'], 'data': api_data_cache['imaging']})
except Exception as e:
traceback.print_exc()
return jsonify('error: {}'.format(e))

@app.route("/api/consort")
def api_consort():
global datetime_format
global api_data_index
global api_data_cache
# try:
if not api_data_index['consort'] or not check_data_current(datetime.strptime(api_data_index['consort'], datetime_format)):
api_date = datetime.now().strftime(datetime_format)
consort_data_json = get_api_consort_data(request)
if consort_data_json:
api_data_cache['consort'] = consort_data_json
api_data_index['consort'] = api_date
return jsonify({'date': api_data_index['consort'], 'data': api_data_cache['consort']})
# except Exception as e:
# traceback.print_exc()
# return jsonify('error: {}'.format(e))

# get_api_consort_data
# @app.route("/api/consort")
# def api_consort():
# global datetime_format
# global api_data_index
# global api_data_cache
# # try:
# if not api_data_index['consort'] or not check_data_current(datetime.strptime(api_data_index['consort'], datetime_format)):
# api_date = datetime.now().strftime(datetime_format)
# consort_data_json = get_api_consort_data(request)
# if consort_data_json:
# api_data_cache['consort'] = consort_data_json
# api_data_index['consort'] = api_date
# return jsonify({'date': api_data_index['consort'], 'data': api_data_cache['consort']})
# # except Exception as e:
# # traceback.print_exc()
# # return jsonify('error: {}'.format(e))

# # get_api_consort_data


@app.route("/api/blood")
def api_blood():
global datetime_format
global api_data_index
global api_data_cache
try:
if not api_data_index['blood'] or not check_data_current(datetime.strptime(api_data_index['blood'], datetime_format)):
api_date = datetime.now().strftime(datetime_format)
blood_data, blood_data_request_status = get_api_blood_data(request)

if data_access_type != 'LOCAL':
data_date = datetime.now().strftime(datetime_format)
blood_data, blood_data_request_status = get_api_blood_data(request)
else:
data_date = local_data_date
blood_data, blood_data_request_status = get_local_blood_data(blood1_filepath, blood2_filepath)

if blood_data:
api_data_index['blood'] = api_date
api_data_index['blood'] = data_date
api_data_cache['blood'] = blood_data

with open('requests.csv', 'a', newline='') as f:
Expand All @@ -196,47 +207,42 @@ def api_subjects():
try:
if not api_data_index['subjects'] or not check_data_current(datetime.strptime(api_data_index['subjects'], datetime_format)):
api_date = datetime.now().strftime(datetime_format)
latest_subjects_json = get_api_subjects_json(request)
if data_access_type != 'LOCAL':
data_date = datetime.now().strftime(datetime_format)
latest_subjects_json = get_api_subjects_json(request)
else:
data_date = local_data_date
latest_subjects_json = get_local_subjects_raw(subjects1_filepath, subjects2_filepath)
if latest_subjects_json:
# latest_data = create_clean_subjects(latest_subjects_json, screening_sites, display_terms_dict, display_terms_dict_multi)
latest_data = create_clean_subjects(latest_subjects_json, screening_sites, display_terms_dict, display_terms_dict_multi)
latest_data = process_subjects_data(latest_subjects_json,subjects_raw_cols_for_reports,screening_sites, display_terms_dict, display_terms_dict_multi)

api_data_cache['subjects'] = latest_data
api_data_index['subjects'] = api_date
api_data_index['subjects'] = data_date
api_data_cache['subjects'] = latest_subjects_json # latest_data


return jsonify({'date': api_data_index['subjects'], 'data': api_data_cache['subjects']})
except Exception as e:
traceback.print_exc()
return jsonify('error: {}'.format(e))
def api_tester():

global local_subjects_data

try:
return jsonify(local_subjects_data)


except Exception as e:
traceback.print_exc()
return jsonify('error: {}'.format(e))


@app.route("/api/full")
def api_full():
datafeeds = {}
for data_category in api_data_cache:
if api_data_cache[data_category]['data']:
datafeeds[data_category] = list(api_data_cache[data_category]['data'].keys())
else:
datafeeds[data_category] = ['no data']
return jsonify(datafeeds)

@app.route("/api/simple")
def api_simple():
if api_data_simple['subjects']:
return jsonify('simple subjects')
else:
return jsonify('not found')
# @app.route("/api/full")
# def api_full():
# datafeeds = {}
# for data_category in api_data_cache:
# if api_data_cache[data_category]['data']:
# datafeeds[data_category] = list(api_data_cache[data_category]['data'].keys())
# else:
# datafeeds[data_category] = ['no data']
# return jsonify(datafeeds)

# @app.route("/api/simple")
# def api_simple():
# if api_data_simple['subjects']:
# return jsonify('simple subjects')
# else:
# return jsonify('not found')


if __name__ == "__main__":
Expand Down
Loading

0 comments on commit ac1b63a

Please sign in to comment.