SIMR/greylag-process

   1 #!/bin/bash
   2
   3 # This wrapper script will invoke greylag in parallel on our SGE cluster.
   4
   5 # This directory is assumed to be shared across all cluster nodes
   6 SHAREDTMPDIR=/clusterfs/home/proteomics/tmp
   7
   8 # The python image to use
   9 PYTHON=/n/site/inst/Linux-i686/sys/bin/python2.4
  10
  11 # greylag lives here (use python2.4 explicitly)
  12 greylag=/clusterfs/site/inst/Linux-i686/bioinfo/greylag/greylag.py
  13
  14 # options passed to all greylag invocations
  15 #greylag_options="-v --quirks-mode"
  16 greylag_options="-v"
  17
  18 # Divide this run into this many parts.  This should probably be at least 5x
  19 # the number of cluster nodes available.  Currently, the fixed overhead cost
  20 # for each part is pretty small (10-20 seconds?).
  21 PARTS=1000
  22
  23
  24 p=`basename $0`
  25
  26 usage() {
  27     cat <<EOF 1>&2
  28 usage: $p [-l] <greylag-params.xml> [ <ms2-file> ... ]
  29
  30 Process a set of ms2 files as specified by the given params file (which must
  31 end in '.xml' and should probably be in the current directory).
  32
  33 If the '-l' flag is given, stdout and stderr will be redirected to a
  34 corresponding '.log' file.
  35
  36 EOF
  37     exit 0
  38 }
  39
  40 err() {
  41     echo 1>&2 "$p: $@"
  42 }
  43
  44 die() {
  45     err "$@"
  46     touch "$jobname.done-failed" || err "touch failed"
  47     exit 1
  48 }
  49
  50 if [ "$1" == "-l" ]; then
  51     shift
  52     logging=1
  53 fi
  54
  55 if [ $# -lt 2 ]; then
  56     usage
  57 fi
  58
  59 params="$1"
  60
  61 rm -f ${params%.xml}.done-*
  62
  63 case "$params" in
  64     *.xml) true;;
  65     *) die "parameter filename must end in '*.xml'";;
  66 esac
  67
  68 jobname=$(basename $params .xml)
  69
  70 shift
  71 for f in "$@"; do
  72     if [ $(dirname $f) != "." ]; then
  73         die "file '$f' must be in the current working directory"
  74     fi
  75     case "$f" in
  76         *.ms2 | *.ms2.gz | *.ms2.bz2) true;;
  77         *) die "argument file '$f' should end in '.ms2' (or '.ms2.{gz,bz2}')";;
  78     esac
  79 done
  80
  81 [ -e $params ] || die "'$params' not found"
  82
  83 if ! ls -ld . | egrep -q '^drwxrws'; then
  84     chmod g+rwxs . || err "attempt to make this directory group writable failed"
  85 fi
  86
  87
  88 # Do some basic locking.  This tries to prevent simultaneous runs on the same
  89 # parameter file, which would produce output to the same file, wasting
  90 # resources and causing confusion.
  91
  92 lockfile="$PWD/$jobname.lock"
  93 trap "rm -f $lockfile" EXIT
  94 ln -s $$ $lockfile 2>/dev/null || true
  95 lockpid=$(ls -ld $lockfile | sed -e 's/^.*> //')
  96
  97 if [ $lockpid != $$ ]; then
  98     die "this directory locked by another process (pid = $lockpid)?
  99          remove $lockfile if not"
 100 fi
 101
 102 # point of no return
 103 if [ "$logging" == "1" ]; then
 104     exec < /dev/null > "$jobname.log" 2>&1
 105 fi
 106
 107 # priority?
 108
 109 # Be very careful with quoting, as these names may eventually come from
 110 # Windows users...
 111
 112 shared_d=$SHAREDTMPDIR/greylag-$(date +%s)-$$ # unique
 113
 114 # Could add removal of the shared directory to the EXIT trap, but probably we
 115 # shouldn't because pdq (and maybe SGE) may react badly
 116 #    trap "rm -rf $lockfile $shared_d &" EXIT
 117
 118 mkdir $shared_d || die "'mkdir $shared_d' failed!"
 119
 120 cp -p "$params" $shared_d/ || die "params cp failed!"
 121
 122 for f in "$@"; do
 123     case "$f" in
 124         *.gz) zcat < "$f" > "$shared_d/$f" || die "ms2 zcat failed";;
 125         *.bz2) bzcat < "$f" > "$shared_d/$f" || die "ms2 zcat failed";;
 126         *) ln -s "$PWD/$f" $shared_d/ || die "ms2 ln -s failed";;
 127     esac
 128 done
 129
 130 results="$jobname.out.xml.gz"
 131
 132 jobbasedir=$(basename $PWD)
 133
 134 #########################################################################
 135 # submit the job in this shared directory so that the nodes can see it
 136 pushd $shared_d > /dev/null || die "pushd failed!"
 137
 138 # use err
 139
 140 #cat <<EOF
 141 #job name: $jobname
 142 #shared dir: $shared_d
 143 #params: $params
 144 #ms2: $@
 145 #EOF
 146 #ls -l $shared_d/
 147
 148 err round 0 setup
 149
 150 # Use of *.ms2 implies that there are no such files here except for the ones
 151 # we just created.  Fix?
 152 # [NB: the ms2 arguments must be given in the same order each time!]
 153
 154 # split ms2 files (on master, for now)
 155 $PYTHON $greylag $greylag_options \
 156     -o "$results" \
 157     --part-split=$PARTS \
 158     "$params" *.ms2 \
 159     || die "round 0 setup on master failed"
 160
 161 err round 0 processing
 162
 163 # The -pe option is currently a hack to try to stop multiple jobs from being
 164 # run on the same node.
 165
 166 # first do parts on nodes
 167 qsub -sync y -r y -b y -cwd -V -hard -l mem_free=250M \
 168     -q all.q \
 169     -t 1-$PARTS \
 170     -N "greylag-$jobbasedir-$jobname-$$" \
 171     -e 'round-1.$TASK_ID.err' \
 172     -o 'round-1.$TASK_ID.out' \
 173     $PYTHON $greylag $greylag_options \
 174     -o "$results" \
 175     --part='${SGE_TASK_ID}of${SGE_TASK_LAST}' \
 176     "$params" *.ms2
 177 qsub_status=$?
 178 cat $(ls -1 round-1.*.{out,err} | sort -t . -k 2n,3) /dev/null
 179 if [ "$qsub_status" != 0 ]; then
 180     die "round 0 on nodes failed"
 181 fi
 182
 183 err round 0 merge
 184
 185 # now merge results (on master, for now)
 186 $PYTHON $greylag $greylag_options \
 187     -o "$results" \
 188     --part-merge=$PARTS \
 189     "$params" *.ms2 \
 190     || die "round 0 merge on master failed"
 191
 192 popd > /dev/null || die "popd failed!"
 193 #########################################################################
 194
 195 if ! [ -e $shared_d/"$results" ]; then
 196     die 'no results file present after processing--job failed?'
 197 fi
 198
 199 cp -p $shared_d/"$results" . || die "cp failed!"
 200
 201 # for now, don't do this
 202 #rm -fr "$shared_d" $lockfile
 203
 204 err complete
 205 touch "$jobname.done-ok" || err "touch failed"
 206 exit 0