diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile new file mode 100644 index 000000000..8b610a4ee --- /dev/null +++ b/ci/Jenkinsfile @@ -0,0 +1,120 @@ +pipeline { + agent none + stages { + stage('UPP Regression Tests') { + agent { + label 'built-in' + } + steps { + script { + for (label in pullRequest.labels) { + if ((label.matches("orion"))) { + env.CHOICE_NODE='orion' + } + else if ((label.matches("hera"))) { + env.CHOICE_NODE='hera' + } + else if ((label.matches("hercules"))) { + env.CHOICE_NODE='hercules' + } + else if ((label.matches("jet"))) { + env.CHOICE_NODE='jet' + } + else { + env.CHOICE_NODE='none' + } + } +// Why do I need another if..block, because it just works this way. + + if (CHOICE_NODE == 'orion') { + echo "Starting up orion ${CHOICE_NODE}...this might take 5-10 minutes...please be patient." + } + else if (CHOICE_NODE == 'jet') { + echo "Starting up jet ${CHOICE_NODE}...this might take 5-10 minutes...please be patient." + } + else if (CHOICE_NODE == 'hercules') { + echo "Starting up hera ${CHOICE_NODE}...this might take 5-10 minutes...please be patient." + } + else if (CHOICE_NODE == 'hera') { + echo "Starting up hera ${CHOICE_NODE}...this might take 5-10 minutes...please be patient." + } + else { + echo "${CHOICE_NODE} is NOT a platform, moving on..." + } + } + } + } + stage('Run UPP RTs on Hera') { + agent { + label "hera" + } + environment { + ACCNR = 'epic' + NODE_PATH = '/scratch2/NAGAPE/epic/UPP/jenkins-ci' + } + steps { + + cleanWs() + checkout scm + sh ''' + echo $(pwd) + cd ci/ + ./rt.sh -a ${ACCNR} -r `pwd`/rundir -t `pwd`/../ + ''' + } + } + stage('Run UPP RTs on Orion') { + agent { + label "orion" + } + environment { + ACCNR = 'epic' + NODE_PATH = '/work/noaa/epic/UPP/jenkins-ci/orion' + } + steps { + + cleanWs() + checkout scm + sh ''' + echo $(pwd) + cd ci/ + ./rt.sh -a ${ACCNR} -r `pwd`/rundir -t `pwd`/../ + ''' + } + } + stage('Run UPP RTs on Hercules') { + agent { + label "hercules" + } + environment { + ACCNR = 'epic' + NODE_PATH = '/work/noaa/epic/UPP/jenkins-ci/hercules' + } + steps { + + cleanWs() + checkout scm + sh ''' + + ''' + } + } + stage('Run UPP RTs on Jet') { + agent { + label "jet" + } + environment { + ACCNR = 'hfv3gfs' + NODE_PATH = '/mnt/lfs4/HFIP/hfv3gfs/role.epic/' + } + steps { + + cleanWs() + checkout scm + sh ''' + + ''' + } + } + } +} diff --git a/ci/rt-status.py b/ci/rt-status.py new file mode 100755 index 000000000..4405639dd --- /dev/null +++ b/ci/rt-status.py @@ -0,0 +1,59 @@ +# check the UPP RT rundir for .diff files that would indicate a change in test results + +''' +Updates +Aug/29/2023 - Fernando Andrade-Maldonado: Script creation + +''' + +import os +import sys + +# files used in result comparison +tests = [ + 'nmmb', + 'gfs', + 'fv3gefs', + 'fv3r', + 'rap', + 'hrrr', + 'fv3hafs', + 'rtma' +] + +# look for .diff files +# every case has its own directory in rundir +# loop through every test case sub directory and files, then match with the test name +def check_for_diff(tests): + changed = False + rundir = os.getenv('rundir') + for case_dir in os.listdir(rundir): + for file in os.listdir(rundir+'/{}'.format(case_dir)): + if file.endswith('.diff'): + diff_case = '' + for test in tests: + if test in case_dir: + diff_case = test + if case_dir.endswith('pe_test'): + # the rap pe test currently has a false positive bug with WRFPRS + if 'rap' in case_dir and file == 'WRFPRS.GrbF16.diff': + with open('{}/{}/{}'.format(rundir,case_dir, file)) as f: + data = f.readlines() + if len(data) == 1 and 'CDCON:convective cloud layer:rpn_corr=-nan:rpn_rms=undefined' in data[0]: + continue + print('There are changes in results for case {}_pe_test in {}'.format(diff_case, file.replace(".diff", ""))) + else: + print('There are changes in results for case {} in {}'.format(diff_case, file.replace(".diff", ""))) + changed = True + if changed: + print('Refer to .diff files in rundir: {} for details on differences in results for each case.'.format(rundir)) + sys.exit(1) + else: + print('No changes in test results detected.') + + +def main(): + check_for_diff(tests) + +if __name__ == "__main__": + main() diff --git a/ci/rt.sh b/ci/rt.sh new file mode 100755 index 000000000..7c25d3f80 --- /dev/null +++ b/ci/rt.sh @@ -0,0 +1,250 @@ +#!/bin/bash +###################################################################### +# This script is desined for UPP regression tests run by UPP developer. +# Wen Meng, 12/2020, First version. +# Fernando Andrade-Maldonado 5/2023 rework for CLI Options +# Fernando Andrade-Maldonado / Wen Meng 9/2023 Add Hercules, fix typos, and refactor +# +###################################################################### + +git_branch="develop" +git_url="https://github.com/NOAA-EMC/UPP.git" +clone_on="no" + +while getopts a:w:h:r:t:b:u:c opt; do + case $opt in + a) accnr=${OPTARG} + ;; + w) workdir=${OPTARG} + ;; + h) homedir=${OPTARG} + ;; + r) rundir=${OPTARG} + ;; + t) test_v=${OPTARG} + ;; + b) git_branch=${OPTARG} + ;; + u) git_url=${OPTARG} + ;; + c) clone_on="yes" + ;; + esac +done + +#UPP working copy +test_v=${test_v:-`pwd`/../} +if [[ $clone_on == "yes" ]]; then + rm -rf $test_v + mkdir -p $test_v + git clone -b $git_branch $git_url $test_v +fi +export svndir=${test_v} + +#Assume a nems account to run with +accnr=${accnr:-"nems"} + +#Build UPP executable +build_exe=yes + +#Choose run specific model +run_nmmb=yes +run_gfs=yes +run_gefs=yes +run_fv3r=yes +run_rap=yes +run_hrrr=yes +run_hafs=yes +run_rtma=yes + +#find machine +mac=$(hostname | cut -c1-1) +mac2=$(hostname | cut -c1-2) +mac3=$(hostname | cut -c1-4) +if [ $mac2 = hf ]; then # for HERA + export machine=HERA + export homedir=${homedir:-"/scratch2/NAGAPE/epic/UPP/test_suite"} + export rundir=${rundir:-"/scratch1/NCEPDEV/stmp2/${USER}"} + module use /scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/modulefiles/stack + module load hpc/1.2.0 + module load hpc-intel/2022.1.2 + module load hpc-impi/2022.1.2 + module load prod_util +elif [ $mac = O ] ; then + export machine=ORION + export homedir=${homedir:-"/work/noaa/epic/UPP"} + export rundir=${rundir:-"/work2/noaa/stmp/$USER"} + module use /work/noaa/epic-ps/role-epic-ps/hpc-stack/libs/intel-2022.1.2/modulefiles/stack + module load hpc/1.2.0 + module load hpc-intel/2022.1.2 + module load hpc-impi/2022.1.2 + module load prod_util/1.2.2 +elif [ $mac3 = herc ] ; then + export machine=HERCULES + export homedir=${homedir:-"/work/noaa/epic/UPP"} + export rundir=${rundir:-"/work2/noaa/stmp/$USER"} + module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-dev-20230717/envs/unified-env/install/modulefiles/Core + module load stack-intel/2021.9.0 + module load stack-intel-oneapi-mpi/2021.9.0 + module load jasper/2.0.32 + module load prod-util/1.2.2 +fi + +#set working directory +export workdir=${workdir:-"`pwd`/work-upp-${machine}"} +rm -rf $workdir +mkdir -p $workdir + +#differentiates for orion and hercules +export rundir="${rundir}/upp-${machine}" + +#set log file +export logfile=`pwd`/rt.log.$machine +if [ -f $logfile ] ; then + rm -r $logfile +fi +runtime_log=$homedir/scripts/runtime.log.$machine + +#build executable +if [ "$build_exe" = "yes" ]; then +cd ${test_v} +mkdir -p ${test_v}/exec +cd ${test_v}/tests +./compile_upp.sh +status=$? +if [ $status -eq 0 ]; then + msg="Building executable successfully" +else + msg="Building executable with failure" + exit +fi +postmsg "$logfile" "$msg" +fi + +jobid_list="" + +#execute nmmb grib2 test +if [ "$run_nmmb" = "yes" ]; then +cd $workdir +cp $homedir/jobs-dev/run_post_nmmb_Grib2_${machine}.sh . +job_id=`sbatch --parsable -A ${accnr} run_post_nmmb_Grib2_${machine}.sh` +jobid_list=$jobid_list" "$job_id +cp $homedir/jobs-dev/run_post_nmmb_Grib2_pe_test_${machine}.sh . +job_id=`sbatch --parsable -A ${accnr} run_post_nmmb_Grib2_pe_test_${machine}.sh` +jobid_list=$jobid_list" "${job_id} +fi + +#execute fv3gefs test +if [ "$run_gefs" = "yes" ]; then +cd $workdir +cp $homedir/jobs-dev/run_post_fv3gefs_${machine}.sh . +job_id=`sbatch --parsable -A ${accnr} run_post_fv3gefs_${machine}.sh` +jobid_list=$jobid_list" "${job_id} +cp $homedir/jobs-dev/run_post_fv3gefs_pe_test_${machine}.sh . +job_id=`sbatch --parsable -A ${accnr} run_post_fv3gefs_pe_test_${machine}.sh` +jobid_list=$jobid_list" "${job_id} +fi + +#execute rap test +if [ "$run_rap" = "yes" ]; then +cd $workdir +cp $homedir/jobs-dev/run_post_rap_${machine}.sh . +job_id=`sbatch --parsable -A ${accnr} run_post_rap_${machine}.sh` +jobid_list=$jobid_list" "$job_id +cp $homedir/jobs-dev/run_post_rap_pe_test_${machine}.sh . +job_id=`sbatch --parsable -A ${accnr} run_post_rap_pe_test_${machine}.sh` +jobid_list=$jobid_list" "${job_id} +fi + +#execute hrrr test +if [ "$run_hrrr" = "yes" ]; then +cd $workdir +cp $homedir/jobs-dev/run_post_hrrr_${machine}.sh . +job_id=`sbatch --parsable -A ${accnr} run_post_hrrr_${machine}.sh` +jobid_list=$jobid_list" "$job_id +cp $homedir/jobs-dev/run_post_hrrr_pe_test_${machine}.sh . +job_id=`sbatch --parsable -A ${accnr} run_post_hrrr_pe_test_${machine}.sh` +jobid_list=$jobid_list" "${job_id} +fi + +#execute fv3gfs test +if [ "$run_gfs" = "yes" ]; then +cd $workdir +cp $homedir/jobs-dev/run_post_fv3gfs_${machine}.sh . +job_id=`sbatch --parsable -A ${accnr} run_post_fv3gfs_${machine}.sh` +jobid_list=$jobid_list" "${job_id} +cp $homedir/jobs-dev/run_post_fv3gfs_pe_test_${machine}.sh . +job_id=`sbatch --parsable -A ${accnr} run_post_fv3gfs_pe_test_${machine}.sh` +jobid_list=$jobid_list" "${job_id} +fi + +#execute fv3r test +if [ "$run_fv3r" = "yes" ]; then +cd $workdir +cp $homedir/jobs-dev/run_post_fv3r_${machine}.sh . +job_id=`sbatch --parsable -A ${accnr} run_post_fv3r_${machine}.sh` +jobid_list=$jobid_list" "${job_id} +cp $homedir/jobs-dev/run_post_fv3r_pe_test_${machine}.sh . +job_id=`sbatch --parsable -A ${accnr} run_post_fv3r_pe_test_${machine}.sh` +jobid_list=$jobid_list" "${job_id} +fi + +#execute fv3hafs test +if [ "$run_hafs" = "yes" ]; then +cd $workdir +cp $homedir/jobs-dev/run_post_fv3hafs_${machine}.sh . +job_id=`sbatch --parsable -A ${accnr} run_post_fv3hafs_${machine}.sh` +jobid_list=$jobid_list" "${job_id} +cp $homedir/jobs-dev/run_post_fv3hafs_pe_test_${machine}.sh . +job_id=`sbatch --parsable -A ${accnr} run_post_fv3hafs_pe_test_${machine}.sh` +jobid_list=$jobid_list" "${job_id} +fi + +#execute rtma test +if [ "$run_rtma" = "yes" ]; then +cd $workdir +cp $homedir/jobs-dev/run_post_3drtma_${machine}.sh . +job_id=`sbatch --parsable -A ${accnr} run_post_3drtma_${machine}.sh` +jobid_list=$jobid_list" "${job_id} +cp $homedir/jobs-dev/run_post_3drtma_pe_test_${machine}.sh . +job_id=`sbatch --parsable -A ${accnr} run_post_3drtma_pe_test_${machine}.sh` +jobid_list=$jobid_list" "${job_id} +fi + +echo "Job cards submitted for enabled tests, waiting on timestamps for finished jobs..." + +#get run time for each test +sleep 30 +for job_id in $jobid_list; do + ic=1 + sleep_loop_max=300 + while [ $ic -le $sleep_loop_max ]; do + job_id=`echo $job_id | cut -d"." -f1` + status=`sacct --parsable -j $job_id --format=jobid,jobname,elapsed,state | cut -d"|" -f4|awk 'FNR == 2'` + if [ "$status" = "COMPLETED" ]; then + break + else + ic=`expr $ic + 1` + sleep 15 + fi + done + if [ $ic -lt $sleep_loop_max ]; then + runtime=`sacct --parsable -j $job_id --format=jobid,jobname,elapsed,state | cut -d"|" -f3|awk 'FNR == 2'` + jobname=`sacct --parsable -j $job_id --format=jobid,jobname,elapsed,state | cut -d"|" -f2|awk 'FNR == 2'` + runtime_b=`grep ${jobname} ${runtime_log} | awk '{print $2}' ` + echo "$runtime $jobname ${runtime_b}" + msg="Runtime: $jobname $runtime -- baseline ${runtime_b}" + postmsg "$logfile" "$msg" + fi +done + +python ${test_v}/ci/rt-status.py + +# Cleanup rt log +cd ${test_v}/ci +echo "rundir: ${rundir}" > rt.log.${machine}.temp +cat rt.log.${machine} | grep "test:" >> rt.log.${machine}.temp +cat rt.log.${machine} | grep "baseline" >> rt.log.${machine}.temp +cat rt.log.${machine}.temp > rt.log.${machine} +rm rt.log.${machine}.temp +mv rt.log.${machine} ${test_v}/tests/logs diff --git a/tests/logs/rt.log.HERA b/tests/logs/rt.log.HERA new file mode 100644 index 000000000..0984bff66 --- /dev/null +++ b/tests/logs/rt.log.HERA @@ -0,0 +1,53 @@ +rundir: /scratch2/NAGAPE/epic/Fernando.Andrade-maldonado/regression-tests/upp/780/UPP/ci/rundir-upp-HERA +10/13 06:21:38Z -hrrr pe test: your new post executable generates bit-identical WRFTWO.GrbF04 as the trunk +10/13 06:21:40Z -hrrr pe test: your new post executable generates bit-identical WRFPRS.GrbF04 as the trunk +10/13 06:21:42Z -hrrr pe test: your new post executable generates bit-identical WRFNAT.GrbF04 as the trunk +10/13 06:21:48Z -fv3gefs test: your new post executable generates bit-identical geaer.t00z.master.grb2f060 as the trunk +10/13 06:22:09Z -fv3r test: your new post executable generates bit-identical PRSLEV10.tm00 as the trunk +10/13 06:22:14Z -rtma pe test: your new post executable generates bit-identical NATLEV00.tm00 as the trunk +10/13 06:22:16Z -fv3r test: your new post executable generates bit-identical NATLEV10.tm00 as the trunk +10/13 06:22:20Z -rtma pe test: your new post executable generates bit-identical PRSLEV00.tm00 as the trunk +10/13 06:22:20Z -rtma pe test: your new post executable generates bit-identical IFIFIP00.tm00 as the trunk +10/13 06:22:36Z -hrrr test: your new post executable generates bit-identical WRFTWO.GrbF04 as the trunk +10/13 06:22:38Z -hrrr test: your new post executable generates bit-identical WRFPRS.GrbF04 as the trunk +10/13 06:22:40Z -hrrr test: your new post executable generates bit-identical WRFNAT.GrbF04 as the trunk +10/13 06:22:45Z -fv3gefs pe test: your new post executable generates bit-identical geaer.t00z.master.grb2f060 as the trunk +10/13 06:22:57Z -fv3hafs test: your new post executable generates bit-identical HURPRS09.tm00 as the trunk +10/13 06:23:09Z -fv3hafs pe test: your new post executable generates bit-identical HURPRS09.tm00 as the trunk +10/13 06:23:26Z -rap test: your new post executable generates bit-identical WRFPRS.GrbF16 as the trunk +10/13 06:23:27Z -rap test: your new post executable generates bit-identical WRFNAT.GrbF16 as the trunk +10/13 06:23:43Z -nmmb pe test: your new post executable generates bit-identical BGDAWP03.tm00.Grib2 as the trunk +10/13 06:23:43Z -rap pe test: your new post executable did generate changed results in WRFPRS.GrbF16 +10/13 06:23:45Z -rap pe test: your new post executable generates bit-identical WRFNAT.GrbF16 as the trunk +10/13 06:23:45Z -nmmb pe test: your new post executable generates bit-identical BGRD3D03.tm00.Grib2 as the trunk +10/13 06:23:45Z -nmmb pe test: your new post executable generates bit-identical BGRDSF03.tm00.Grib2 as the trunk +10/13 06:23:48Z -nmmb test: your new post executable generates bit-identical BGDAWP03.tm00.Grib2 as the trunk +10/13 06:23:49Z -nmmb test: your new post executable generates bit-identical BGRD3D03.tm00.Grib2 as the trunk +10/13 06:23:49Z -nmmb test: your new post executable generates bit-identical BGRDSF03.tm00.Grib2 as the trunk +10/13 06:24:00Z -fv3r pe test: your new post executable generates bit-identical PRSLEV10.tm00 as the trunk +10/13 06:24:05Z -fv3r pe test: your new post executable generates bit-identical NATLEV10.tm00 as the trunk +10/13 06:24:09Z -rtma test: your new post executable generates bit-identical NATLEV00.tm00 as the trunk +10/13 06:24:13Z -rtma test: your new post executable generates bit-identical PRSLEV00.tm00 as the trunk +10/13 06:24:13Z -rtma test: your new post executable generates bit-identical IFIFIP00.tm00 as the trunk +10/13 06:31:06Z gfs_post_00.127117-fv3gfs test: your new post executable generates bit-identical gfs.t00z.master.grb2f006 as the trunk +10/13 06:31:07Z gfs_post_00.127117-fv3gfs test: your new post executable generates bit-identical gfs.t00z.sfluxgrbf006.grib2 as the trunk +10/13 06:31:07Z gfs_post_00.127117-fv3gfs test: your new post executable generates bit-identical gfs.t00z.special.grb2f006 as the trunk +10/13 06:31:45Z gfs_post_00.104523-fv3gfs pe test: your new post executable generates bit-identical gfs.t00z.master.grb2f006 as the trunk +10/13 06:31:46Z gfs_post_00.104523-fv3gfs pe test: your new post executable generates bit-identical gfs.t00z.sfluxgrbf006.grib2 as the trunk +10/13 06:31:46Z gfs_post_00.104523-fv3gfs pe test: your new post executable generates bit-identical gfs.t00z.special.grb2f006 as the trunk +10/13 06:24:01Z -Runtime: nmmb_test 00:01:17 -- baseline 00:01:00 +10/13 06:24:01Z -Runtime: nmmb_pe_test 00:01:12 -- baseline 00:01:00 +10/13 06:24:01Z -Runtime: fv3gefs_test 00:00:22 -- baseline 00:40:00 +10/13 06:24:02Z -Runtime: fv3gefs_pe_test 00:00:24 -- baseline 00:40:00 +10/13 06:24:02Z -Runtime: rap_test 00:01:04 -- baseline 00:02:00 +10/13 06:24:02Z -Runtime: rap_pe_test 00:01:12 -- baseline 00:02:00 +10/13 06:24:03Z -Runtime: hrrr_test 00:02:16 -- baseline 00:02:00 +10/13 06:24:03Z -Runtime: hrrr_pe_test 00:02:18 -- baseline 00:02:00 +10/13 06:31:22Z -Runtime: fv3gfs_test 00:12:09 -- baseline 00:15:00 +10/13 06:31:52Z -Runtime: fv3gfs_pe_test 00:12:47 -- baseline 00:15:00 +10/13 06:31:53Z -Runtime: fv3r_test 00:01:51 -- baseline 00:03:00 +10/13 06:31:53Z -Runtime: fv3r_pe_test 00:01:51 -- baseline 00:03:00 +10/13 06:31:53Z -Runtime: fv3hafs_test 00:00:39 -- baseline 00:03:00 +10/13 06:31:54Z -Runtime: fv3hafs_pe_test 00:00:41 -- baseline 00:03:00 +10/13 06:31:54Z -Runtime: rtma_test 00:01:59 -- baseline 00:03:00 +10/13 06:31:54Z -Runtime: rtma_test_pe_test 00:01:56 -- baseline diff --git a/tests/logs/rt.log.ORION b/tests/logs/rt.log.ORION new file mode 100644 index 000000000..d30ddfa5d --- /dev/null +++ b/tests/logs/rt.log.ORION @@ -0,0 +1,53 @@ +rundir: /work2/noaa/epic/nandoam/regression-testing/upp/orion/780/UPP/ci/rundir-upp-ORION +10/23 20:13:51Z -fv3gefs test: your new post executable generates bit-identical geaer.t00z.master.grb2f060 as the trunk +10/23 20:13:55Z -fv3gefs pe test: your new post executable generates bit-identical geaer.t00z.master.grb2f060 as the trunk +10/23 20:14:11Z -fv3hafs test: your new post executable generates bit-identical HURPRS09.tm00 as the trunk +10/23 20:14:11Z -fv3hafs pe test: your new post executable generates bit-identical HURPRS09.tm00 as the trunk +10/23 20:14:40Z -rap test: your new post executable generates bit-identical WRFPRS.GrbF16 as the trunk +10/23 20:14:40Z -rap pe test: your new post executable did generate changed results in WRFPRS.GrbF16 +10/23 20:14:41Z -rap pe test: your new post executable generates bit-identical WRFNAT.GrbF16 as the trunk +10/23 20:14:41Z -rap test: your new post executable generates bit-identical WRFNAT.GrbF16 as the trunk +10/23 20:15:06Z -nmmb pe test: your new post executable generates bit-identical BGDAWP03.tm00.Grib2 as the trunk +10/23 20:15:07Z -nmmb pe test: your new post executable generates bit-identical BGRD3D03.tm00.Grib2 as the trunk +10/23 20:15:08Z -nmmb pe test: your new post executable generates bit-identical BGRDSF03.tm00.Grib2 as the trunk +10/23 20:15:12Z -fv3r test: your new post executable generates bit-identical PRSLEV10.tm00 as the trunk +10/23 20:15:14Z -fv3r pe test: your new post executable generates bit-identical PRSLEV10.tm00 as the trunk +10/23 20:15:16Z -fv3r test: your new post executable generates bit-identical NATLEV10.tm00 as the trunk +10/23 20:15:18Z -fv3r pe test: your new post executable generates bit-identical NATLEV10.tm00 as the trunk +10/23 20:15:24Z -rtma pe test: your new post executable generates bit-identical NATLEV00.tm00 as the trunk +10/23 20:15:25Z -nmmb test: your new post executable generates bit-identical BGDAWP03.tm00.Grib2 as the trunk +10/23 20:15:26Z -rtma pe test: your new post executable generates bit-identical PRSLEV00.tm00 as the trunk +10/23 20:15:26Z -nmmb test: your new post executable generates bit-identical BGRD3D03.tm00.Grib2 as the trunk +10/23 20:15:27Z -nmmb test: your new post executable generates bit-identical BGRDSF03.tm00.Grib2 as the trunk +10/23 20:15:27Z -rtma pe test: your new post executable generates bit-identical IFIFIP00.tm00 as the trunk +10/23 20:15:30Z -rtma test: your new post executable generates bit-identical NATLEV00.tm00 as the trunk +10/23 20:15:32Z -rtma test: your new post executable generates bit-identical PRSLEV00.tm00 as the trunk +10/23 20:15:33Z -rtma test: your new post executable generates bit-identical IFIFIP00.tm00 as the trunk +10/23 20:15:39Z -hrrr pe test: your new post executable generates bit-identical WRFTWO.GrbF04 as the trunk +10/23 20:15:40Z -hrrr pe test: your new post executable generates bit-identical WRFPRS.GrbF04 as the trunk +10/23 20:15:42Z -hrrr pe test: your new post executable generates bit-identical WRFNAT.GrbF04 as the trunk +10/23 20:21:03Z -hrrr test: your new post executable generates bit-identical WRFTWO.GrbF04 as the trunk +10/23 20:21:04Z -hrrr test: your new post executable generates bit-identical WRFPRS.GrbF04 as the trunk +10/23 20:21:06Z -hrrr test: your new post executable generates bit-identical WRFNAT.GrbF04 as the trunk +10/23 20:25:06Z gfs_post_00.31515-fv3gfs pe test: your new post executable generates bit-identical gfs.t00z.master.grb2f006 as the trunk +10/23 20:25:07Z gfs_post_00.31515-fv3gfs pe test: your new post executable generates bit-identical gfs.t00z.sfluxgrbf006.grib2 as the trunk +10/23 20:25:07Z gfs_post_00.31515-fv3gfs pe test: your new post executable generates bit-identical gfs.t00z.special.grb2f006 as the trunk +10/23 20:25:32Z gfs_post_00.442830-fv3gfs test: your new post executable generates bit-identical gfs.t00z.master.grb2f006 as the trunk +10/23 20:25:33Z gfs_post_00.442830-fv3gfs test: your new post executable generates bit-identical gfs.t00z.sfluxgrbf006.grib2 as the trunk +10/23 20:25:33Z gfs_post_00.442830-fv3gfs test: your new post executable generates bit-identical gfs.t00z.special.grb2f006 as the trunk +10/23 20:15:32Z -Runtime: nmmb_test 00:01:56 -- baseline 00:03:00 +10/23 20:15:33Z -Runtime: nmmb_pe_test 00:01:37 -- baseline 00:03:00 +10/23 20:15:33Z -Runtime: fv3gefs_test 00:00:20 -- baseline 01:20:00 +10/23 20:15:33Z -Runtime: fv3gefs_pe_test 00:00:24 -- baseline 01:20:00 +10/23 20:15:34Z -Runtime: rap_test 00:01:10 -- baseline 00:02:00 +10/23 20:15:34Z -Runtime: rap_pe_test 00:01:10 -- baseline 00:02:00 +10/23 20:21:07Z -Runtime: hrrr_test 00:07:35 -- baseline 00:02:00 +10/23 20:21:07Z -Runtime: hrrr_pe_test 00:02:11 -- baseline 00:02:00 +10/23 20:25:39Z -Runtime: fv3gfs_test 00:12:03 -- baseline 00:18:00 +10/23 20:25:40Z -Runtime: fv3gfs_pe_test 00:11:36 -- baseline 00:18:00 +10/23 20:25:40Z -Runtime: fv3r_test 00:01:46 -- baseline 00:03:00 +10/23 20:25:40Z -Runtime: fv3r_pe_test 00:01:47 -- baseline 00:03:00 +10/23 20:25:41Z -Runtime: fv3hafs_test 00:00:40 -- baseline 00:00:40 +10/23 20:25:41Z -Runtime: fv3hafs_pe_test 00:00:41 -- baseline 00:00:40 +10/23 20:25:41Z -Runtime: rtma_test 00:02:03 -- baseline 00:04:00 +10/23 20:25:41Z -Runtime: rtma_pe_test 00:01:57 -- baseline 00:04:00