3 '''Run a greylag job split across multiple local processes. This is not for
4 use on a cluster, but useful if you have just one multi-CPU machine, or for
9 greylag, Copyright (C) 2006-2007, Stowers Institute for Medical Research
11 This program is free software; you can redistribute it and/or modify
12 it under the terms of the GNU General Public License as published by
13 the Free Software Foundation; either version 2 of the License, or
14 (at your option) any later version.
16 This program is distributed in the hope that it will be useful,
17 but WITHOUT ANY WARRANTY; without even the implied warranty of
18 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 GNU General Public License for more details.
21 You should have received a copy of the GNU General Public License along
22 with this program; if not, write to the Free Software Foundation, Inc.,
23 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
27 # This might better be implemented as a shell script under Unix. It's done in
28 # Python here as a demonstration, and so that greylag can be easily run on
29 # multiple CPUs under Windows, as X!Tandem can.
31 # This is not intended to be used on cluster nodes. In particular, like
32 # X!Tandem, it just divides the spectra into N parts and then processes them
33 # separately, making no further attempt at load balancing.
38 from socket
import gethostname
43 GREYLAG_PROGRAM
= 'greylag'
47 print >> sys
.stderr
, 'warning:', s
49 sys
.exit('error: ' + s
)
52 print >> sys
.stderr
, ('Usage: %s <processes> <greylag-options-and-args>...'
54 '%s\n(see "greylag.py --help" for more information)'
55 % (os
.path
.basename(sys
.argv
[0]), __doc__
))
59 def run_parts(processes
, partprefix
, args
):
60 ret
= subprocess
.call([GREYLAG_PROGRAM
, '--part-split=%s' % processes
,
61 '--part-prefix=%s' % partprefix
] + args
)
63 error("part split failed")
65 # list of currently running subprocess.Popen objects
68 for n
in range(processes
):
69 p
= subprocess
.Popen([GREYLAG_PROGRAM
,
70 '--part=%sof%s' % (n
+1, processes
),
71 '--part-prefix=%s' % partprefix
] + args
)
76 raise EnvironmentError("error status")
77 except EnvironmentError, e
:
78 error("part process failed [%s]" % e
)
80 # upon error, try to kill any remaining processes
84 os
.kill(p
.pid
, signal
.SIGINT
)
88 ret
= subprocess
.call([GREYLAG_PROGRAM
, '--part-merge=%s' % processes
,
89 '--part-prefix=%s' % partprefix
] + args
)
91 error("part merge failed")
96 if len(args
) < 2 or '-h' in args
or '--help' in args
:
99 processes
= int(args
[0])
103 error("<processes> must be a positive integer")
104 # This limit can be raised by setting environment variable.
105 GREYLAGPROCESSES
= os
.environ
.get('GREYLAGPROCESSES', 8)
106 if processes
> GREYLAGPROCESSES
:
107 error("current limit is %s processes (see source code for more)"
109 if any(x
for x
in args
if x
.startswith('--part')):
110 error("no --part options may be specified")
112 # try to generate a unique prefix, to avoid (unlikely) collision
113 partprefix
= '#greylag-part-%s-%s' % (gethostname(), os
.getpid())
116 run_parts(processes
, partprefix
, args
[1:])
118 # try to clean up temporary part files
119 for fn
in os
.listdir('.'):
120 if fn
.startswith(partprefix
):
124 warn("could not remove '%s' [%s]" % (fn
, e
))
127 if __name__
== '__main__':