src/arch/bluegenep/charmrun

   1 #!/bin/sh
   2 #
   3 # Conv-host for bluegenep:
   4 #  Translates +pN-style conv-host options into
   5 # qsub options.
   6 # The script qsub the job, wait until it finishes, and print the output.
   7 # good for job less than 30 minutes
   8
   9 args=""
  10 pes=1
  11 machinefile=""
  12 QUIET=0
  13
  14 while [ $# -gt 0 ]
  15 do
  16         case $1 in
  17         +ppn)
  18                 args=$args" +ppn "$2
  19                 shift
  20                 ;;
  21         +ppn*)
  22                 args=$args" "$1
  23                 ;;
  24         +p)
  25                 pes=$2
  26                 shift
  27                 ;;
  28         +p*)
  29                 pes=`echo $1 | awk '{print substr($1,3)}'`
  30                 ;;
  31         -machinefile)
  32                 machinefile=$2
  33                 args=" "$1" "$2" "$args
  34                 shift
  35                 ;;
  36         --mode)
  37                 args=" "$1" "$2" "$args
  38                 shift
  39                 ;;
  40         ++quiet)
  41                 QUIET=1
  42                 args=$args" "$1
  43                 ;;
  44         *)
  45                 args=$args" "$1
  46                 ;;
  47         esac
  48         shift
  49 done
  50
  51
  52 test $QUIET -eq 0 && printf "\nRunning on $pes processors: $args\n"
  53
  54 if test -n "$COBALT_JOBID"
  55 then
  56   # charmrun called from script
  57   test $QUIET -eq 0 && echo "Running> cobalt-mpirun -nofree -env BG_MAXALIGNEXP=0 -np $pes $args"
  58   cobalt-mpirun -nofree -env BG_MAXALIGNEXP=0 -np $pes $args
  59 else
  60
  61 queue_stat=qstat
  62 queue_kill=qdel
  63 queue_sub=qsub
  64
  65 while [ true ]
  66 do
  67
  68 test $QUIET -eq 0 && echo "Submitting batch job for> $pes $args"
  69 test $QUIET -eq 0 && echo " using the command> $queue_sub -t 30 -n $pes  $args"
  70 jobid=""
  71 while [ -z "$jobid" ]
  72 do
  73   jobid=`$queue_sub -t 30 -n $pes  $args 2>err.$$ |tail -1`
  74   if grep 'not found' err.$$ > /dev/null
  75   then
  76     cat err.$$
  77     rm -f err.$$
  78     exit 1
  79   fi
  80   sleep 10
  81 done
  82 test $QUIET -eq 0 && echo "Job enqueued under job ID $jobid"
  83
  84 output=$jobid.output
  85 err=$jobid.error
  86
  87 End() {
  88         echo "Charmrun> $queue_kill $jobid ..."
  89         $queue_kill $jobid
  90         rm -f $script
  91         exit $1
  92 }
  93
  94 # kill job if interrupted
  95 trap 'End 1' 2 3
  96
  97 # Wait for the job to complete, by checking its status
  98 while [ true ]
  99 do
 100         $queue_stat $jobid > tmp.$$
 101         exitstatus=$?
 102
 103         if test $exitstatus -ne 0
 104         then
 105             # job not in the queue now
 106             status=`tail -1 $err | sed 's/.*Exit status:[ ]*\([0-9\]*\).*/\1/'`
 107             if test -n $status
 108             then
 109                   if test $status = 1
 110                   then
 111                     if grep 'Failed to boot the partition' $err > /dev/null 2>/dev/null
 112                     then
 113                       echo "Failed to boot the partition, retrying."
 114                       break
 115                     fi
 116                   fi
 117                   cat $output
 118                   rm -f tmp.$$
 119                   exit $status
 120             fi
 121         fi
 122         sleep 20
 123 done
 124
 125 done
 126
 127 exit 1
 128
 129 fi