diff --git a/deploy/default/kevin-woudc-api-config.yml b/deploy/default/kevin-woudc-api-config.yml old mode 100644 new mode 100755 index 2fd9eb7..da8588c --- a/deploy/default/kevin-woudc-api-config.yml +++ b/deploy/default/kevin-woudc-api-config.yml @@ -18,7 +18,7 @@ server: logging: level: DEBUG - logfile: /net/local/home/ngaik/woudc-api_env/woudc-api/kevin-woudc-api.log + logfile: /tmp/kevin-woudc-api.log metadata: identification: diff --git a/deploy/default/woudc-api-config.yml b/deploy/default/woudc-api-config.yml old mode 100644 new mode 100755 index c019fcd..c3c24db --- a/deploy/default/woudc-api-config.yml +++ b/deploy/default/woudc-api-config.yml @@ -18,7 +18,7 @@ server: logging: level: ERROR - # logfile: /tmp/pygeoapi.log + # logfile: /tmp/woudc-api.log metadata: identification: diff --git a/deploy/nightly/deploy-nightly.sh b/deploy/nightly/deploy-nightly.sh old mode 100644 new mode 100755 index 928bebe..c531f9a --- a/deploy/nightly/deploy-nightly.sh +++ b/deploy/nightly/deploy-nightly.sh @@ -1,8 +1,10 @@ +#!/bin/bash + # ================================================================= # -# Author: Tom Kralidis +# Author: Kevin Ngai # -# Copyright (c) 2021 Tom Kralidis +# Copyright (c) 2024 Kevin Ngai # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -40,57 +42,83 @@ export WOUDC_API_BIND_PORT=5000 export WOUDC_API_ES_URL=https://${WOUDC_API_ES_USERNAME}:${WOUDC_API_ES_PASSWORD}@localhost:9200 export WOUDC_API_OGC_SCHEMAS_LOCATION=/data/web/woudc-api-nightly/latest/schemas.opengis.net +DATETIME=$(date +%Y%m%d) +TIMESTAMP=$(date +%Y%m%d.%H%M) +NIGHTLYDIR="woudc-api-$TIMESTAMP" + +log() { + echo "$(date +'%Y-%m-%d %H:%M:%S') - $*" +} + +cleanup_old_builds() { + log "Deleting nightly builds > $DAYSTOKEEP days old" + find . -type d -name "woudc-api-20*" | while read -r dir; do + DATETIME2=$(echo "$dir" | awk -F- '{print $3}' | awk -F. '{print $1}') + DIFF=$(( ( $(date +%s -d "$DATETIME") - $(date +%s -d "$DATETIME2") ) / 86400 )) + if [ "$DIFF" -gt "$DAYSTOKEEP" ]; then + rm -rf "$dir" + fi + done + rm -rf latest +} + +create_venv_and_install() { + log "Generating nightly build for $TIMESTAMP" + python3 -m venv --system-site-packages "$NIGHTLYDIR" && cd "$NIGHTLYDIR" || exit 1 + source bin/activate + + log "Cloning repositories..." + git clone "$WOUDC_API_GITREPO" + git clone "$WOUDC_EXTCSV_GITREPO" + git clone "$PYGEOAPI_GITREPO" + + log "Installing pygeoapi..." + cd pygeoapi || exit 1 + git checkout 0.16.1 + pip3 install -r requirements.txt + python3 setup.py install + + log "Installing woudc-extcsv..." + cd ../woudc-extcsv || exit 1 + python3 setup.py install + + log "Installing woudc-api..." + cd ../woudc-api || exit 1 + python3 setup.py install + cd .. +} + +configure_woudc_api() { + log "Generating schemas.opengis.net..." + mkdir schemas.opengis.net + curl -O http://schemas.opengis.net/SCHEMAS_OPENGIS_NET.zip + unzip ./SCHEMAS_OPENGIS_NET.zip "ogcapi/*" -d schemas.opengis.net + rm -f ./SCHEMAS_OPENGIS_NET.zip + + log "Configuring woudc-api configurations..." + cp woudc-api/deploy/default/woudc-api-config.yml woudc-api/deploy/nightly + sed -i 's#basepath: /#basepath: /woudc-api/nightly/latest#' woudc-api/deploy/nightly/woudc-api-config.yml + sed -i 's^# cors: true^cors: true^' woudc-api/deploy/nightly/woudc-api-config.yml + + log "Generating woudc-api-openapi.yml..." + pygeoapi openapi generate woudc-api/deploy/nightly/woudc-api-config.yml > woudc-api/deploy/nightly/woudc-api-openapi.yml + sed -i "s#http://schemas.opengis.net#$WOUDC_API_URL/schemas#g" woudc-api/deploy/nightly/woudc-api-openapi.yml +} + +set_symlink_and_permissions() { + log "Creating 'latest' symlink and setting correct permissions..." + ln -s "$NIGHTLYDIR" latest + chgrp eccc-hpc-cmdx -R "$NIGHTLYDIR" + chmod -R 775 "$NIGHTLYDIR" + log "Done." +} + +main() { + cd "$BASEDIR" || exit 1 + cleanup_old_builds + create_venv_and_install + configure_woudc_api + set_symlink_and_permissions +} -# you should be okay from here - -DATETIME=`date +%Y%m%d` -TIMESTAMP=`date +%Y%m%d.%H%M` -NIGHTLYDIR=woudc-api-$TIMESTAMP - -echo "Deleting nightly builds > $DAYSTOKEEP days old" - -cd $BASEDIR - -for f in `find . -type d -name "woudc-api-20*"` -do - DATETIME2=`echo $f | awk -F- '{print $3}' | awk -F. '{print $1}'` - let DIFF=(`date +%s -d $DATETIME`-`date +%s -d $DATETIME2`)/86400 - if [ $DIFF -gt $DAYSTOKEEP ]; then - rm -fr $f - fi -done - -rm -fr latest -echo "Generating nightly build for $TIMESTAMP" -python3.8 -m venv --system-site-packages $NIGHTLYDIR && cd $NIGHTLYDIR -source bin/activate -git clone $WOUDC_API_GITREPO -git clone $WOUDC_EXTCSV_GITREPO -git clone $PYGEOAPI_GITREPO -cd pygeoapi -pip3.8 install --prefix $BASEDIR/$NIGHTLYDIR cython -pip3.8 install --prefix $BASEDIR/$NIGHTLYDIR "click >= 7.1" pyproj==1.9.6 -pip3.8 install --prefix $BASEDIR/$NIGHTLYDIR -r requirements.txt -pip3.8 install --prefix $BASEDIR/$NIGHTLYDIR flask_cors elasticsearch -python3.8 setup.py install -cd ../woudc-extcsv -python3.8 setup.py install -cd ../woudc-api -python3.8 setup.py install -cd .. - -mkdir schemas.opengis.net -curl -O http://schemas.opengis.net/SCHEMAS_OPENGIS_NET.zip && unzip ./SCHEMAS_OPENGIS_NET.zip "ogcapi/*" -d schemas.opengis.net && rm -f ./SCHEMAS_OPENGIS_NET.zip - -cp woudc-api/deploy/default/woudc-api-config.yml woudc-api/deploy/nightly -sed -i 's#basepath: /#basepath: /woudc-api/nightly/latest#' woudc-api/deploy/nightly/woudc-api-config.yml -sed -i 's^# cors: true^cors: true^' woudc-api/deploy/nightly/woudc-api-config.yml - -pygeoapi openapi generate woudc-api/deploy/nightly/woudc-api-config.yml > woudc-api/deploy/nightly/woudc-api-openapi.yml -sed -i "s#http://schemas.opengis.net#$WOUDC_API_URL/schemas#g" woudc-api/deploy/nightly/woudc-api-openapi.yml - -cd .. - -ln -s $NIGHTLYDIR latest -chgrp dmsec -R $NIGHTLYDIR # ensure correct group permission -chmod -R 775 $NIGHTLYDIR # ensure group writable +main "$@"