bin/foamCheckJobs

   1 #!/bin/sh
   2 #------------------------------------------------------------------------------
   3 # =========                 |
   4 # \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
   5 #  \\    /   O peration     |
   6 #   \\  /    A nd           | Copyright (C) 1991-2008 OpenCFD Ltd.
   7 #    \\/     M anipulation  |
   8 #-------------------------------------------------------------------------------
   9 # License
  10 #     This file is part of OpenFOAM.
  11 #
  12 #     OpenFOAM is free software; you can redistribute it and/or modify it
  13 #     under the terms of the GNU General Public License as published by the
  14 #     Free Software Foundation; either version 2 of the License, or (at your
  15 #     option) any later version.
  16 #
  17 #     OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
  18 #     ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  19 #     FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  20 #     for more details.
  21 #
  22 #     You should have received a copy of the GNU General Public License
  23 #     along with OpenFOAM; if not, write to the Free Software Foundation,
  24 #     Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  25 #
  26 # Script
  27 #     foamCheckJobs
  28 #
  29 # Description
  30 #     Uses runningJobs/, finishedJobs/ and foamProcessInfo to create stateFile.
  31 #     stateFile contains per pid information on state of process. Format:
  32 #       pid state command
  33 #
  34 #     where state is one of 'RUNN', 'SUSP', 'OTHR', 'FINI', 'ABRT' ('PEND')
  35 #     (first three are from foamProcessInfo, others from jobInfo files)
  36 #     (PEND is special state from when user has submitted but no jobInfo
  37 #     file yet. Not supported by this script yet)
  38 #
  39 #------------------------------------------------------------------------------
  40
  41 PROGNAME=${0##*/}
  42
  43 #-------------------------------------------------------------------------------
  44 #- User settings
  45
  46 #- Number of days for files to be considered old
  47 NDAYSLIMIT=7
  48 #-------------------------------------------------------------------------------
  49
  50 #- work file
  51 TMPFILE=/tmp/${PROGNAME}$$.tmp
  52 #- work dir. Needs to be accessible for all machines
  53 MACHDIR=$HOME/.FreeFOAM/${PROGNAME}
  54 DEFSTATEFILE=$HOME/.FreeFOAM/foamCheckJobs.out
  55
  56
  57 if [ `uname -s` = 'Linux' ]; then
  58    ECHO='echo -e'
  59 else
  60    ECHO='echo'
  61 fi
  62
  63 #-------------------------------------------------------------------------------
  64 #
  65 # Functions
  66 #
  67 #-------------------------------------------------------------------------------
  68
  69 . $WM_PROJECT_DIR/tools/jobControlFunctions
  70
  71 # dayDiff <date string 1>  <date string 2>
  72 # Prints number of days between the two
  73 # Eg. dayDiff "Jan 10 2002" "Dec 28 1999"
  74 #     ==> 13
  75 dayDiff() {
  76     date -d "$1" > /dev/null 2>&1
  77     if [ $? -ne 0 ]; then
  78         #- option '-d' on date not supported. Give up.
  79         echo "0"
  80     else
  81         year1=`echo "$1" | awk '{print $3}'`
  82         year2=`echo "$2" | awk '{print $3}'`
  83         day1=`date -d "$1" "+%j"`
  84         day2=`date -d "$2" "+%j"`
  85
  86         nYears=`expr $year1 - $year2`
  87         tmp1=`expr $nYears \* 365`
  88         tmp2=`expr $day1 - $day2`
  89         expr $tmp1 + $tmp2
  90     fi
  91 }
  92 #dayDiff  "`date '+%b %d %Y'`" "Dec 28 2001"
  93
  94
  95 # getAllJobs jobInfoDirectory
  96 # Prints list of all jobs in directory (e.g. runningJobs/)
  97 # Also handles 'slaves' entries in jobInfo:
  98 #   slaves           1 ( penfold.23766 );
  99 getAllJobs() {
 100     if notEmpty $1; then
 101         jobs=$1/*
 102         for f in $jobs
 103         do
 104             line=`grep '^[ ]*slaves' $f 2>/dev/null`
 105             if [ $? -eq 0 ]; then
 106                 slaveJobs=`echo "$line" | sed -e 's/.*(\(.*\)).*/\1/'`
 107                 jobs="$jobs $slaveJobs"
 108             fi
 109         done
 110     else
 111         jobs=''
 112     fi
 113     echo "$jobs"
 114 }
 115
 116 # releaseLock jobId lockFile
 117 # Releases lock on jobId
 118 releaseLock () {
 119     if [ -f $2 ]; then
 120         #- move lock to finishedJobs
 121         mv $2 $FOAM_JOB_DIR/finishedJobs/
 122     fi
 123     $ECHO "Lock on job $1 released."
 124 }
 125
 126
 127 printUsage() {
 128 cat << LABEL
 129 Usage: $PROGNAME [stateFile]
 130
 131 This program checks all the locks in the license directory to see if
 132 their processes are still running. Processes will not release their
 133 lock if they exit abnormally. This program will try to obtain process
 134 information on the machine the process ran on and release the lock
 135 if the program is no longer running.
 136
 137 Requirements: the environment variable FOAM_JOB_DIR needs to point to the
 138 license directory and all machines have to be reachable using ssh.
 139
 140 The output from checking all running jobs is collected in an optional
 141 file.
 142
 143 FILES:
 144     FOAM_JOB_DIR/runningJobs    locks for running processes
 145                   /finishedJobs     ,,      finished processes
 146
 147     where FOAM_JOB_DIR is determined by (in that order):
 148     - value of the \$FREEFOAM_JOB_DIR environment variable if defined
 149     - value of the \$FOAM_JOB_DIR environment variable if defined
 150       (for backward compatibility)
 151    - value of the "foamJobDir" entry in the global \`controlDict' file
 152    - \$HOME/.FreeFOAM/jobControl
 153
 154 LABEL
 155 }
 156
 157
 158 #-------------------------------------------------------------------------------
 159 #
 160 # Main
 161 #
 162 #-------------------------------------------------------------------------------
 163
 164 FOAM_JOB_DIR=`foamJobDir`
 165
 166 #- Check a few things
 167
 168 if [ ! "$FOAM_JOB_DIR" ]; then
 169     $ECHO "$PROGNAME : FOAM_JOB_DIR variable not set."
 170     $ECHO "This indicates an error in this script."
 171     exit 1
 172 fi
 173
 174 if [ ! -d "$FOAM_JOB_DIR" ]; then
 175     $ECHO "$PROGNAME : The license directory $FOAM_JOB_DIR is not valid."
 176     exit 1
 177 fi
 178 if [ ! -d "$FOAM_JOB_DIR/runningJobs" -o ! -d "$FOAM_JOB_DIR/finishedJobs" ]; then
 179     $ECHO "$PROGNAME : The license directory $FOAM_JOB_DIR is not valid."
 180     exit 1
 181 fi
 182
 183 if [ $# -eq 1 ]; then
 184     STATEFILE=$1
 185 elif [ $# -eq 0 ]; then
 186     STATEFILE=${STATEFILE:-$DEFSTATEFILE}
 187 else
 188     printUsage
 189     exit 1
 190 fi
 191
 192 #- obtain rsh method
 193 RSH='ssh'
 194 echo "Using remote shell type : $RSH"
 195
 196 echo ""
 197 echo "Collecting information on jobs in"
 198 echo "  $FOAM_JOB_DIR"
 199 echo ""
 200
 201
 202 #- Collect machine names into $TMPFILE
 203 # Also handles 'slaves' entry in jobInfo:
 204
 205 rm -f $TMPFILE; touch $TMPFILE
 206 RUNJOBS=`getAllJobs $FOAM_JOB_DIR/runningJobs`
 207 for f in $RUNJOBS
 208 do
 209     machinePid=`basename $f`
 210     machine=`echo $machinePid | sed -e 's/\.[0-9][0-9]*$//'`
 211     pid=`echo $machinePid | sed -e 's/.*\.\([0-9][0-9]*\)$/\1/'`
 212
 213     fgrep "$machine" $TMPFILE >/dev/null 2>&1
 214     if [ $? -ne 0 ]; then
 215         $ECHO "$machine" >> $TMPFILE
 216     fi
 217 done
 218 $ECHO "Found machines:"
 219 cat $TMPFILE
 220 $ECHO ""
 221
 222
 223
 224 #- Collect process info on all machines, one file per machine
 225
 226 mkdir -p $MACHDIR
 227 cnt=1
 228 while true
 229 do
 230     machine=`sed -n -e "${cnt}p" $TMPFILE`
 231     if [ ! "$machine" ]; then
 232         break
 233     fi
 234
 235     machFile=$MACHDIR/$machine
 236     rm -f $machFile
 237     $ECHO "Contacting $machine to collect process information:"
 238     if [ $machine = `hostname` ]; then
 239         $ECHO "  foamProcessInfo $machFile"
 240         foamProcessInfo $machFile >/dev/null 2>&1
 241     else
 242         $ECHO "  $RSH $machine foamProcessInfo $machFile"
 243         $RSH $machine foamProcessInfo $machFile >/dev/null 2>&1
 244     fi
 245     if [ $? -ne 0 -o ! -s $machFile ]; then
 246         $ECHO "** Failed collecting process information on $machine."
 247         $ECHO "Check $machFile and run foamProcessInfo by hand"
 248         rm -f $machFile
 249     else
 250         $ECHO "Succesfully collected information in $machFile ..."
 251     fi
 252
 253     cnt=`expr $cnt + 1`
 254 done
 255 $ECHO ""
 256
 257
 258 #- Construct state for runningJobs; move non runnning jobs to finishedJobs
 259
 260 releaseAll=''
 261 rm -f $STATEFILE
 262 for f in $RUNJOBS
 263 do
 264     machinePid=`basename $f`
 265     machine=`echo $machinePid | sed -e 's/\.[0-9][0-9]*$//'`
 266     pid=`echo $machinePid | sed -e 's/.*\.\([0-9][0-9]*\)$/\1/'`
 267
 268     machFile=$MACHDIR/$machine
 269     if [ -r $machFile ]; then
 270         entry=`grep "^$pid " $machFile 2>/dev/null`
 271         if [ $? -ne 0 -o ! "$entry" ]; then
 272             if [ "$releaseAll" ]; then
 273                 releaseLock $machinePid $f
 274             else
 275                 $ECHO "Job $machinePid seems to be no longer running. Release lock? (y/a)\c"
 276                 read answ
 277                 if [ "${answ:-y}" = 'y' ]; then
 278                     releaseLock $machinePid $f
 279                 elif [ "${answ:-y}" = 'a' ]; then
 280                     releaseAll='yes'
 281                     releaseLock $machinePid $f
 282                 else
 283                     state='OTHR'
 284                     $ECHO "$machinePid $state" >> $STATEFILE
 285                 fi
 286             fi
 287         else
 288             state=`echo "$entry" | awk '{print $2}'`
 289             $ECHO "$machinePid $state" >> $STATEFILE
 290         fi
 291     fi
 292 done
 293
 294
 295
 296 #- Collect old jobs in finishedJobs
 297
 298 OLDFILES=`find $FOAM_JOB_DIR/finishedJobs -mtime +$NDAYSLIMIT -print`
 299
 300 #- Construct state for finishedJobs and check on date of files.
 301
 302 if notEmpty $FOAM_JOB_DIR/finishedJobs; then
 303     dateNow=`date '+%b %d %Y'`
 304     for f in $FOAM_JOB_DIR/finishedJobs/*
 305     do
 306         sz=`ls -s $f | awk '{print $1}'`
 307         if [ "$sz" -gt 0 ]; then
 308             machinePid=`basename $f`
 309             machine=`echo $machinePid | sed -e 's/\.[0-9][0-9]*$//'`
 310             pid=`echo $machinePid | sed -e 's/.*\.\([0-9][0-9]*\)$/\1/'`
 311
 312             end=`getEntry $f endDate`
 313             if [ ! "$end" ]; then
 314                 state='ABRT'
 315             else
 316                 nDaysOld=`dayDiff  "$dateNow" "$end"`
 317                 if [ "$nDaysOld" -gt $NDAYSLIMIT ]; then
 318                     OLDFILES="$OLDFILES $f"
 319                 fi
 320
 321                 state='FINI'
 322             fi
 323
 324             $ECHO "$machinePid $state" >> $STATEFILE
 325         fi
 326     done
 327 fi
 328
 329
 330 #- Remove old locks
 331
 332 nOldFiles=`echo "$OLDFILES" | wc -w`
 333 if [ "$nOldFiles" -gt 0 ]; then
 334     $ECHO "You seem to have $nOldFiles locks older than $NDAYSLIMIT days in finishedJobs/"
 335     $ECHO "Do you want to remove these? (y)\c"
 336     read answ
 337     if [ "${answ:-y}" = 'y' ]; then
 338         rm -f $OLDFILES
 339     fi
 340 fi
 341
 342
 343 rm -f $TMPFILE
 344 rm -r $MACHDIR
 345
 346 $ECHO ""
 347 $ECHO "Updated stateFile:"
 348 $ECHO "  $STATEFILE"
 349 $ECHO ""
 350
 351 # ------------------------- vim: set sw=4 sts=4 et: --------------- end-of-file