greylag_solo.py

   1 #!/usr/bin/env greylag-python
   2
   3 '''Run a greylag job split across multiple local processes.  This is not for
   4 use on a cluster, but useful if you have just one multi-CPU machine, or for
   5 debugging.
   6
   7 If you specify more processes than the total number of spectra in the input
   8 files, you will get harmless "no input spectra" warnings, which may be
   9 ignored.
  10 '''
  11
  12 __copyright__ = '''
  13     greylag, Copyright (C) 2006-2007, Stowers Institute for Medical Research
  14
  15     This program is free software; you can redistribute it and/or modify
  16     it under the terms of the GNU General Public License as published by
  17     the Free Software Foundation; either version 2 of the License, or
  18     (at your option) any later version.
  19
  20     This program is distributed in the hope that it will be useful,
  21     but WITHOUT ANY WARRANTY; without even the implied warranty of
  22     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  23     GNU General Public License for more details.
  24
  25     You should have received a copy of the GNU General Public License along
  26     with this program; if not, write to the Free Software Foundation, Inc.,
  27     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  28 '''
  29
  30
  31 # This might better be implemented as a shell script under Unix.  It's done in
  32 # Python here as a demonstration, and so that greylag can be easily run on
  33 # multiple CPUs under Windows.
  34
  35 # This is not intended to be used on cluster nodes.  In particular, it just
  36 # naively divides the spectra into N parts and processes them separately,
  37 # making no further attempt at load balancing.
  38
  39
  40 import os
  41 import os.path
  42 from socket import gethostname
  43 import subprocess
  44 import sys
  45
  46
  47 def warn(s):
  48     print >> sys.stderr, 'warning:', s
  49 def error(s):
  50     sys.exit('error: ' + s)
  51
  52 def usage():
  53     print >> sys.stderr, ('Usage: %s <processes>'
  54                           ' <greylag-grind-options-and-args>...'
  55                           '\n\n'
  56                           '%s\n(see "greylag-grind --help" for more'
  57                           ' information)'
  58                           % (os.path.basename(sys.argv[0]), __doc__))
  59     sys.exit()
  60
  61
  62 def slices(N):
  63     """Generate N equal slices.
  64
  65     >>> list(slices(4))
  66     [(0.0, 0.25), (0.25, 0.5), (0.5, 0.75), (0.75, 1.0)]
  67
  68     """
  69
  70     s = 1.0 / N
  71     for i in range(N):
  72         w0 = i * s
  73         w1 = (i+1) * s
  74         if i == N-1:
  75             w1 = 1.0                    # forestall rounding issues
  76         yield (round(w0,3), round(w1,3))
  77
  78
  79 def run_parts_and_merge(processes, job_id, args):
  80     merge_fn = 'greylag-merge-%s.gwr' % job_id
  81     work_fns = [ 'grind_%s_%s-%s.gwr' % (job_id, w0, w1)
  82                  for w0, w1 in slices(processes) ]
  83
  84     try:
  85         # list of currently running subprocess.Popen objects
  86         subprocs = []
  87         try:
  88             for w0, w1 in slices(processes):
  89                 p = subprocess.Popen(['greylag-grind', '--job-id='+job_id,
  90                                       '-w', str(w0), str(w1)] + args)
  91                 subprocs.append(p)
  92             while subprocs:
  93                 p = subprocs.pop()
  94                 if p.wait():
  95                     raise EnvironmentError("error status")
  96         except EnvironmentError, e:
  97             error("greylag-grind process failed [%s]" % e)
  98         finally:
  99             # upon error, try to kill any remaining processes
 100             try:
 101                 import signal
 102                 for p in subprocs:
 103                     os.kill(p.pid, signal.SIGINT)
 104             except Exception:
 105                 pass
 106
 107         ret = subprocess.call(['greylag-merge'] + work_fns + [merge_fn])
 108         if ret:
 109             error("greylag-merge failed")
 110
 111         ret = subprocess.call(['greylag-sqt', merge_fn])
 112         if ret:
 113             error("greylag-sqt failed")
 114     finally:
 115         try:
 116             os.remove(merge_fn)
 117             for fn in work_fns:
 118                 os.remove(fn)
 119         except OSError:
 120             pass
 121
 122
 123 def main(args=sys.argv[1:]):
 124     if len(args) < 3 or '-h' in args or '--help' in args:
 125         usage()
 126     try:
 127         processes = int(args[0])
 128         if processes < 1:
 129             raise ValueError
 130     except ValueError:
 131         error("<processes> must be a positive integer")
 132
 133     # Keep naive users out of trouble.  This limit can be raised by setting
 134     # environment variable.
 135     GREYLAGPROCESSES = os.environ.get('GREYLAGPROCESSES', 8)
 136     if processes > GREYLAGPROCESSES:
 137         error("current limit is %s processes (see source code for more)"
 138               % GREYLAGPROCESSES)
 139
 140     for a in args:
 141         if a.startswith('--job-id'):
 142             error('--job-id may not be specified')
 143         if a.startswith(('-w', '--work-slice')):
 144             error('--work-slice may not be specified')
 145
 146     # try to generate a unique prefix, to avoid (unlikely) collision
 147     job_id = 'solo-%s-%s' % (gethostname(), os.getpid())
 148
 149     run_parts_and_merge(processes, job_id, args[1:])
 150
 151
 152 if __name__ == '__main__':
 153     main()