1 #!/usr/bin/env greylag-python
3 '''Run a greylag job split across multiple local processes. This is not for
4 use on a cluster, but useful if you have just one multi-CPU machine, or for
7 If you specify more processes than the total number of spectra in the input
8 files, you will get harmless "no input spectra" warnings, which may be
13 greylag, Copyright (C) 2006-2007, Stowers Institute for Medical Research
15 This program is free software; you can redistribute it and/or modify
16 it under the terms of the GNU General Public License as published by
17 the Free Software Foundation; either version 2 of the License, or
18 (at your option) any later version.
20 This program is distributed in the hope that it will be useful,
21 but WITHOUT ANY WARRANTY; without even the implied warranty of
22 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23 GNU General Public License for more details.
25 You should have received a copy of the GNU General Public License along
26 with this program; if not, write to the Free Software Foundation, Inc.,
27 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
31 # This might better be implemented as a shell script under Unix. It's done in
32 # Python here as a demonstration, and so that greylag can be easily run on
33 # multiple CPUs under Windows.
35 # This is not intended to be used on cluster nodes. In particular, it just
36 # naively divides the spectra into N parts and processes them separately,
37 # making no further attempt at load balancing.
42 from socket
import gethostname
48 print >> sys
.stderr
, 'warning:', s
50 sys
.exit('error: ' + s
)
53 print >> sys
.stderr
, ('Usage: %s <processes>'
54 ' <greylag-grind-options-and-args>...'
56 '%s\n(see "greylag-grind --help" for more'
58 % (os
.path
.basename(sys
.argv
[0]), __doc__
))
63 """Generate N equal slices.
66 [(0.0, 0.25), (0.25, 0.5), (0.5, 0.75), (0.75, 1.0)]
75 w1
= 1.0 # forestall rounding issues
76 yield (round(w0
,3), round(w1
,3))
79 def run_parts_and_merge(processes
, job_id
, args
):
80 merge_fn
= 'greylag-merge-%s.gwr' % job_id
81 work_fns
= [ 'grind_%s_%s-%s.gwr' % (job_id
, w0
, w1
)
82 for w0
, w1
in slices(processes
) ]
85 # list of currently running subprocess.Popen objects
88 for w0
, w1
in slices(processes
):
89 p
= subprocess
.Popen(['greylag-grind', '--job-id='+job_id
,
90 '-w', str(w0
), str(w1
)] + args
)
95 raise EnvironmentError("error status")
96 except EnvironmentError, e
:
97 error("greylag-grind process failed [%s]" % e
)
99 # upon error, try to kill any remaining processes
103 os
.kill(p
.pid
, signal
.SIGINT
)
107 ret
= subprocess
.call(['greylag-merge'] + work_fns
+ [merge_fn
])
109 error("greylag-merge failed")
111 ret
= subprocess
.call(['greylag-sqt', merge_fn
])
113 error("greylag-sqt failed")
123 def main(args
=sys
.argv
[1:]):
124 if len(args
) < 3 or '-h' in args
or '--help' in args
:
127 processes
= int(args
[0])
131 error("<processes> must be a positive integer")
133 # Keep naive users out of trouble. This limit can be raised by setting
134 # environment variable.
135 GREYLAGPROCESSES
= os
.environ
.get('GREYLAGPROCESSES', 8)
136 if processes
> GREYLAGPROCESSES
:
137 error("current limit is %s processes (see source code for more)"
141 if a
.startswith('--job-id'):
142 error('--job-id may not be specified')
143 if a
.startswith(('-w', '--work-slice')):
144 error('--work-slice may not be specified')
146 # try to generate a unique prefix, to avoid (unlikely) collision
147 job_id
= 'solo-%s-%s' % (gethostname(), os
.getpid())
149 run_parts_and_merge(processes
, job_id
, args
[1:])
152 if __name__
== '__main__':