6 # This script runs a large number of WRF regression tests. On the
7 # IBM machines (which at NCAR allow queueing), the jobs are sent to
8 # the queue system via load leveler commands. On all other machines,
9 # the jobs are sent in sequentially as soon as the process returns
10 # from completing the previous task.
17 # mkdir /data3/mp/$USER/`hostname`
18 # put all_reg.csh, regtest.csh, and wrf.tar in dir
21 # flex lm errors show up as fails to compile
24 # bluevista or blueice
25 # put all_reg.csh, regtest.csh, and wrf.tar in ~
29 # Unless you are editing the script, no changes are required below
32 #=======================================================================
33 #=======================================================================
35 # The only really important thing.
37 if ( ! -e wrf.
tar ) then
39 echo "Whoa there pardner, where is that wrf.tar file"
44 # What these tests do, must be a single string.
46 set NAME
= ( "Standard" "NESTED=FALSE" "NESTED=FALSE" "NONE" 1 \
47 "Moving_Nest" "NESTED=FALSE" "NESTED=TRUE" "NONE" 2 \
48 "Full_Optimization" "REG_TYPE=BIT4BIT" "REG_TYPE=OPTIMIZED" "NONE" 3 \
49 "Chemistry" "CHEM=FALSE" "CHEM=TRUE" "NONE" 4 \
50 "Chemistry2" "KPP=FALSE" "KPP=TRUE" "NONE" 5 \
51 "Quilting" "QUILT=FALSE" "QUILT=TRUE" "NONE" 6 \
52 "Binary_IO" "IO_FORM=2" "IO_FORM=1" "NONE" 7 \
53 "GriB1_Output" "IO_FORM=2" "IO_FORM=5" "NONE" 8 \
54 "REAL8_Floats" "REAL8=FALSE" "REAL8=TRUE" "NONE" 9 \
55 "FDDA" "FDDA=FALSE" "FDDA=TRUE" "NONE" 10 \
56 "FDDA2" "FDDA2=FALSE" "FDDA2=TRUE" "NONE" 11 \
57 "ESMF_Library" "ESMF_LIB=FALSE" "ESMF_LIB=TRUE" "ONLY_AIX" 12 \
58 "Global" "GLOBAL=FALSE" "GLOBAL=TRUE" "NONE" 13 \
59 "Adaptive" "ADAPTIVE=FALSE" "ADAPTIVE=TRUE" "NONE" 14 \
62 # Where are we located.
64 set starting_dir
= `pwd`
66 # Get the tag manually. This is for the auto-report that gets
67 # sent to the WRF web page on the history of the regression tests.
69 set current_year4
= `date -u +"%Y"`
70 set current_month
= `date -u +"%m"`
71 set current_day
= `date -u +"%d"`
72 set current_hour
= `date -u +"%H"`
73 set current_minute
= `date -u +"%M"`
74 set current_second
= `date -u +"%S"`
75 set datehms
= ${current_year4}-${current_month}-${current_day}_${current_hour}:${current_minute}:${current_second}_UTC
76 set dateh
= ${current_year4}${current_month}${current_day}${current_hour}
77 set date = ${current_year4}${current_month}${current_day}
79 if ( $user == michalak
) then
81 else if ( $user == hender
) then
83 else if ( $user == gill
) then
85 else if ( $user == dudhia
) then
87 else if ( $user == weiwang
) then
92 echo the TAG is NOT defined
93 echo Please define an environment variable that is the WRFV2 tag
94 echo Something such as
: setenv TAG trunk_
${date}_
${initials}
99 # Are we only interested in the PASS/FAIL report?
101 if ( $#argv == 1 ) then
102 if ( $argv[1] == PASSFAIL
) then
107 # If there are any command line args, they are processed, else
108 # we run all of the regression tests without a generate or
109 # compare flag being set.
111 if ( $#argv == 0 ) then
113 set BASELINE
= RUN_ONLY
121 while ( $count_test < $#NAME )
122 set tests
= ( $tests "$NAME[$count_test]" )
124 set OLD_TEXT
= ( $OLD_TEXT "$NAME[$count_test]" )
126 set NEW_TEXT
= ( $NEW_TEXT "$NAME[$count_test]" )
128 set TOAST
= ( $TOAST "$NAME[$count_test]" )
130 set TEST_NUM
= ( $TEST_NUM "$NAME[$count_test]" )
134 # We have some command line args. They are either a request to run
135 # the test with a generate/compare flag, or a list of tests to perform.
139 set INIT_OPTS
= ( $
* )
141 # First, find the baseline type. This is going to be one of three
142 # possibilities: GENERATE, COMPARE, or a RUN_ONLY option. The
143 # default is RUN_ONLY. Only the first baseline option found is
147 while ( $count < $#INIT_OPTS )
149 set arg
= $INIT_OPTS[$count]
150 if ( ( $arg == GENERATE
) || \
151 ( $arg == COMPARE
) || \
152 ( $arg == RUN_ONLY
) ) then
154 goto FINISHED_BASELINE_TYPE
157 set BASELINE
= RUN_ONLY
158 FINISHED_BASELINE_TYPE
:
160 # If there was only one input, and it was telling us to do which type of baseline
161 # option (GENERATE, COMPARE, RUN_ONLY), we assume they want all of the tests
164 if ( $#INIT_OPTS == 1 ) then
165 if ( ( $INIT_OPTS == GENERATE
) ||
( $INIT_OPTS == COMPARE
) ||
( $INIT_OPTS == RUN_ONLY
) ) then
172 while ( $count_test < $#NAME )
173 set tests
= ( $tests "$NAME[$count_test]" )
175 set OLD_TEXT
= ( $OLD_TEXT "$NAME[$count_test]" )
177 set NEW_TEXT
= ( $NEW_TEXT "$NAME[$count_test]" )
179 set TOAST
= ( $TOAST "$NAME[$count_test]" )
181 set TEST_NUM
= ( $TEST_NUM "$NAME[$count_test]" )
184 goto FINISHED_TEST_LIST
188 # Find which tests are to be conducted. Loop over all of the
189 # input, and compare each of the input fields with the list
190 # of available test names. When a match occurs, increment the
191 # test found counter, and save the test name.
199 while ( $count < $#INIT_OPTS )
202 while ( $count_test < $#NAME )
203 if ( $INIT_OPTS[$count] == $NAME[$count_test] ) then
204 set tests
= ( $tests "$NAME[$count_test]" )
206 set OLD_TEXT
= ( $OLD_TEXT "$NAME[$count_test]" )
208 set NEW_TEXT
= ( $NEW_TEXT "$NAME[$count_test]" )
210 set TOAST
= ( $TOAST "$NAME[$count_test]" )
212 set TEST_NUM
= ( $TEST_NUM "$NAME[$count_test]" )
220 # If there are no recognizable tests requested, let them know our concern.
222 if ( $#tests == 0 ) then
223 echo No valid
test requested
in argument list
229 # A friendly check for the baseline directory existence, and locations for known
232 if ( ( $BASELINE == GENERATE
) ||
( $BASELINE == COMPARE
) ) then
233 if ( ( `uname` == AIX
) && ( ( `hostname | cut -c 1-2` != bs
) && \
234 ( `hostname | cut -c 1-2` != bv
) && ( `hostname | cut -c 1-2` != bl
) ) ) then
235 set SAVE_DIR
= /ptmp
/${USER}/BASELINE
/`uname`
236 else if ( `uname` == AIX
) then
237 set SAVE_DIR
= /ptmp
/${USER}/BASELINE
/`uname`
238 else if ( ( `uname` == Darwin
) && ( `hostname` == stink
) ) then
239 set SAVE_DIR
= /stink
/${user}/Regression_Tests
/BASELINE
/`uname`
240 else if ( ( `uname` == Linux
) && ( `hostname` == bay-mmm
) ) then
241 set SAVE_DIR
= /data
3/mp
/${USER}/BASELINE
/`uname`
243 echo Hmm
, no idea where to put
/get this baseline data
, stopping
247 if ( ( -d $SAVE_DIR ) && ( $BASELINE == GENERATE
) ) then
248 echo "Troubles with SAVE_DIR logic."
249 echo "$SAVE_DIR should not exist for a $BASELINE run."
251 else if ( ( ! -d $SAVE_DIR ) && ( $BASELINE == COMPARE
) ) then
252 echo "Troubles with SAVE_DIR logic."
253 echo "$SAVE_DIR should exist for a $BASELINE run."
258 # We need the regtest.csh file, badly.
260 if ( ! -e regtest.csh
) then
261 echo we need regtest.csh
in this dir
262 echo "supply-ez vous, s'il vous plait"
266 # The regtest.csh file is treated as a template. The following
267 # strings (first occurrence) is sought (OLD_TEXT) and replaced
268 # with the modified string (NEW_TEXT). This edited regression
269 # script is then processed.
271 # Any exceptions to where they can run? NONE means no
272 # exceptions, should run on all machines. AIX/Linux
273 # means it will NOT run on that single machine. The
274 # option ONLY_AIX/ONLY_Linux means that the option
275 # ONLY works on that specific architecture.
278 # Loop over all selected tests.
281 while ( $count_test < $#tests )
286 while ( $count < $#NAME )
288 if ( "$tests[$count_test]" == "$NAME[$count]" ) then
289 goto FOUND_SELECTED_TEST
293 echo "Hmmm, no valid test found"
298 # Specifically skip this test on this architecture.
300 if ( `uname` == $TOAST[$count_test] ) then
301 echo skipping
test $tests[$count_test] for `uname` specifically
303 # Skip this test on this architecture because it ONLY runs
304 # on a different architecture.
306 else if ( ( `echo $TOAST[$count_test] | cut -c 1-5` == ONLY_
) && ( ONLY_
`uname` != $TOAST[$count_test] ) ) then
307 echo skipping
test $tests[$count_test] for `uname`, works on
$TOAST[$count_test]
309 # OK, we are allowed to run this test on this architecture.
311 else if ( ( $TOAST[$count_test] == NONE
) || \
312 ( $TOAST[$count_test] != `uname` ) || \
313 ( ONLY_
`uname` == $TOAST[$count_test] ) ) then
314 echo doing
test $tests[$count_test] for `uname`
316 # If this is the generate or compare baseline test, where do we
317 # save the data to/read the data from.
319 if ( ( $BASELINE == GENERATE
) ||
( $BASELINE == COMPARE
) ) then
320 if ( ( `uname` == AIX
) && ( ( `hostname | cut -c 1-2` != bs
) && \
321 ( `hostname | cut -c 1-2` != bv
) && ( `hostname | cut -c 1-2` != bl
) ) ) then
322 set SAVE_DIR
= /ptmp
/${USER}/BASELINE
/`uname`/$tests[$count_test]
323 else if ( `uname` == AIX
) then
324 set SAVE_DIR
= /ptmp
/${USER}/BASELINE
/`uname`/$tests[$count_test]
325 else if ( ( `uname` == Darwin
) && ( `hostname` == stink
) ) then
326 set SAVE_DIR
= /stink
/${USER}/Regression_Tests
/BASELINE
/`uname`/$tests[$count_test]
327 else if ( ( `uname` == Linux
) && ( `hostname` == bay-mmm
) ) then
328 set SAVE_DIR
= /data
3/mp
/${USER}/BASELINE
/`uname`/$tests[$count_test]
330 echo No idea where to put the data
, stopping
334 # Either zap existing stuff (GENERATE), or make sure it is there (COMPARE)
336 if ( $BASELINE == GENERATE
) then
337 /bin
/rm -rf $SAVE_DIR
338 else if ( ( $BASELINE == COMPARE
) && ( $tests[$count_test] != Full_Optimization
) ) then
339 if ( ! -d $SAVE_DIR ) then
340 echo $SAVE_DIR does not exist
for BASELINE comparison
, stopping
343 else if ( ( $BASELINE == COMPARE
) && ( $tests[$count_test] == Full_Optimization
) ) then
344 echo No comparison
done with baseline since this is an optimized run
348 # Build the short edit input script for ed and edit the regtest.csh file.
350 set OLDT
= `echo $OLD_TEXT[$count_test] | sed 's/=/ = /'`
351 set NEWT
= `echo $NEW_TEXT[$count_test] | sed 's/=/ = /'`
353 if ( ( $BASELINE == RUN_ONLY
) ||
( $tests[$count_test] == Full_Optimization
) ) then
354 if ( -e ed.
in ) rm ed.
in
357 w reg.foo.$TEST_NUM[$count_test].$tests[$count_test]
360 else if ( $BASELINE == GENERATE
) then
361 if ( -e ed.
in ) rm ed.
in
364 ,s?GENERATE_BASELINE = FALSE?GENERATE_BASELINE = $SAVE_DIR?
365 w reg.foo.$TEST_NUM[$count_test].$tests[$count_test]
368 else if ( $BASELINE == COMPARE
) then
369 if ( -e ed.
in ) rm ed.
in
372 ,s?COMPARE_BASELINE = FALSE?COMPARE_BASELINE = $SAVE_DIR?
373 w reg.foo.$TEST_NUM[$count_test].$tests[$count_test]
377 ed regtest.csh
< ed_in
>& /dev
/null
378 chmod +x reg.foo.
$TEST_NUM[$count_test].
$tests[$count_test]
380 # On AIX, we submit jobs to the load leveler queue for bluesky and to the LSF queue for
381 # bluevista. After submission, we wait around until it completes, and then we send in
384 if ( ( `uname` == AIX
) && ( `hostname | cut -c 1-2` == bs
) ) then
385 llsubmit reg.foo.
$TEST_NUM[$count_test].
$tests[$count_test] >&! llsub.out
389 sleep 10 ; llq
-u $USER >&! llq.report
390 grep `cat llsub.out | grep '"bs1' | cut -d\" -f2` llq.report
>& /dev
/null
392 if ( ( $ok == 0 ) && ( $in_already == 0 ) ) then
394 set joe_id
= `cat llsub.out | grep '"bs1' | cut -d\" -f2 | cut -d. -f2`
397 cp /ptmp
/$USER/wrf_regression.
$joe_id/wrftest.output wrftest.output.
$TEST_NUM[$count_test].
$tests[$count_test]
398 rm llsub.out llq.report
399 else if ( ( `uname` == AIX
) && ( ( `hostname | cut -c 1-2` == bv
) ||
( `hostname | cut -c 1-2` == bl
) ) ) then
400 bsub
< reg.foo.
$TEST_NUM[$count_test].
$tests[$count_test] >&! bsub.out
404 sleep 10 ; bjobs
>&! bjobs.report
405 grep `cat bsub.out | grep Job | cut -d"<" -f2 | cut -d">" -f1` bjobs.report
>& /dev
/null
407 if ( ( $ok == 0 ) && ( $in_already == 0 ) ) then
409 set joe_id
= `cat bsub.out | grep Job | cut -d"<" -f2 | cut -d">" -f1`
412 cp /ptmp
/$USER/wrf_regression.
$joe_id/wrftest.output wrftest.output.
$TEST_NUM[$count_test].
$tests[$count_test]
413 rm bsub.out bjobs.report
414 else if ( ( `uname` == AIX
) && ( ( `hostname | cut -c 1-2` != bs
) && \
415 ( `hostname | cut -c 1-2` != bv
) && ( `hostname | cut -c 1-2` != bl
) ) ) then
416 llsubmit reg.foo.
$TEST_NUM[$count_test].
$tests[$count_test] >&! llsub.out
420 sleep 10 ; llq
-u $USER >&! llq.report
421 set llsubmit_name_all
= `cat llsub.out | grep '"b' | cut -d\" -f2`
422 set llsubmit_name_front
= `echo $llsubmit_name_all | cut -d\. -f1`
423 set llsubmit_name_end
= `echo $llsubmit_name_all | cut -d\. -f5`
424 grep $llsubmit_name_front llq.report |
grep $llsubmit_name_end >& /dev
/null
426 if ( ( $ok == 0 ) && ( $in_already == 0 ) ) then
428 set joe_id
= `cat llsub.out | grep '"b' | cut -d\" -f2 | cut -d. -f5`
431 cp /ptmp
/$USER/wrf_regression.
$joe_id/wrftest.output wrftest.output.
$TEST_NUM[$count_test].
$tests[$count_test]
432 rm llsub.out llq.report
434 # On the "other" non-queued machines, we just execute the script and wait until
435 # we get the process returning control, then we move on.
438 reg.foo.
$TEST_NUM[$count_test].
$tests[$count_test] -f wrf.
tar >&! output.
$TEST_NUM[$count_test].
$tests[$count_test]
439 mv wrf_regression
/wrftest.output wrftest.output.
$TEST_NUM[$count_test].
$tests[$count_test]
440 if ( -d wrf_regression
) then
441 mv wrf_regression wrf_regression.
$TEST_NUM[$count_test].
$tests[$count_test]
447 if ( ( `uname` == AIX
) && ( ( `hostname | cut -c 1-2` != bs
) && \
448 ( `hostname | cut -c 1-2` != bv
) && ( `hostname | cut -c 1-2` != bl
) ) ) then
449 echo no web page building
, stopping
455 # Build the html page. We only need the middle portion. It's
456 # a table with 5 columns: Date of test, WRFV2 tag, Developer
457 # who conducted the test, machine the test was run on, and the
458 # pass/fail status of the all_reg.csh script when compared
459 # to the benchmark results (usually a released code).
461 cat >! history_middle_OK.html
<< EOF
466 <td> XARCHITECTUREX </td>
467 <td BGCOLOR="#00FF00"><a href="ARCHITECTURE/wrftest.all_${datehms}">PASS</a>
468 <a href="ARCHITECTURE/message_${datehms}">DIFFS</a></td>
471 <a href="other_docs/${date}_notes">Descriptions</a>
472 <a href="other_docs/${date}_minutes">Minutes</a>
478 cat >! history_middle_OOPS.html
<< EOF
483 <td> XARCHITECTUREX </td>
484 <td BGCOLOR="#FF0000"><a href="ARCHITECTURE/wrftest.all_${datehms}">FAIL</a>
485 <a href="ARCHITECTURE/message_${datehms}">DIFFS</a></td>
488 <a href="other_docs/${date}_notes">Descriptions</a>
489 <a href="other_docs/${date}_minutes">Minutes</a>
497 ,s/ARCHITECTURE/${name}/g
498 w history_middle.html
502 # Get all of the wrftest.output.* files in one BIG file.
504 if ( -e wrftest.all_
$datehms ) rm wrftest.all_
$datehms
505 cat wrftest.output.?.
* wrftest.output.??.
* >>! wrftest.all_
$datehms
506 if ( -d `uname` ) rm -rf `uname`
508 mv wrftest.all_
$datehms `uname`
510 # Compare regression PASS/FAILs with previous runs.
512 pushd ~gill
/RESULTS
/`uname`
513 grep FAIL wrftest.output.
* >! ${starting_dir}/PREV.FAILS
515 grep FAIL wrftest.output.
* >! CURR.FAILS
516 echo Comparison of regression results on
`hostname` for `date` >! message
517 echo "==================================================================" >> message
519 echo "Previous FAILs" >> message
520 echo "==================" >> message
522 cat PREV.FAILS
>> message
524 echo "Current FAILs" >> message
525 echo "==================" >> message
527 cat CURR.FAILS
>> message
529 echo "Difference of FAILs" >> message
530 echo "==================" >> message
532 cat CURR.FAILS |
grep -vi baseline
>! CURR2.FAILS
533 diff PREV.FAILS CURR2.FAILS
>! diffs
537 cp message message_
$datehms
538 mv message_
$datehms `uname`
540 # Send out status info on the regression test.
544 if ( $#argv != 0 ) then
550 if ( ( $NUMARGS == 0 ) ||
( $ARG1 == PASSFAIL
) || \
551 ( ( $ARG1 == COMPARE
) && ( $NUMARGS == 1 ) ) || \
552 ( ( $ARG1 == GENERATE
) && ( $NUMARGS == 1 ) ) ) then
554 Mail
-s "REG DIFFS $OS" ${user}@ucar.edu
< message
556 echo "Different FAILS from before for $OS - repository may have been broken"
558 m4 -DXDATEX=${dateh} -DXTAGX=${TAG} -DXTESTERX=${user} -DXARCHITECTUREX=`uname` \
559 history_middle_OOPS.html
>! history_middle_1.html
560 else if ( $ok == 0 ) then
561 Mail
-s "REG OK $OS" ${user}@ucar.edu
< message
563 echo "Same FAILS as before for $OS - repository OK"
565 m4 -DXDATEX=${dateh} -DXTAGX=${TAG} -DXTESTERX=${user} -DXARCHITECTUREX=`uname` \
566 history_middle_OK.html
>! history_middle_1.html
569 # Store on the NCAR MSS system to circumvent security.
571 ed history_middle_1.html
< ed2.
in >& /dev
/null
572 tar -cf `uname`_hist.
tar history_middle.html
`uname`
573 echo Storing info on NCAR MSS
574 msrcp
`uname`_hist.
tar mss
:/GILL
/ALLREG
/`uname`_hist.
tar
575 if ( -e `uname`_NEW
) rm `uname`_NEW
576 echo "`uname` `date`" >! `uname`_NEW
577 msrcp
`uname`_NEW mss
:/GILL
/ALLREG
/`uname`_NEW
580 rm ed_in
>& /dev
/null
581 rm io_format
>& /dev
/null
582 rm PREV.FAILS CURR.FAILS CURR2.FAILS message diffs
>& /dev
/null
583 rm history_middle_OOPS.html history_middle_OK.html history_middle_1.html history_middle.html
>& /dev
/null
584 rm `uname`_hist.
tar `uname`_NEW ed2.
in >& /dev
/null
585 rm -rf `uname` >& /dev
/null