src/misc/series_mpi.py

   1 #!/usr/bin/env mpipython
   2
   3
   4 #   gpiv_series - Processes a set of numbered input data
   5
   6 #   Copyright (C) 2008 Gerber van der Graaf
   7
   8 #   This program is free software; you can redistribute it and/or modify
   9 #   it under the terms of the GNU General Public License as published by
  10 #   the Free Software Foundation; either version 2, or (at your option)
  11 #   any later version.
  12
  13 #   This program is distributed in the hope that it will be useful,
  14 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 #   GNU General Public License for more details.
  17
  18 #   You should have received a copy of the GNU General Public License
  19 #   along with this program; if not, write to the Free Software Foundation,
  20 #   Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  21
  22
  23 #--------------------------------------------------------------------
  24
  25 #
  26 # This version is MPI enabled for parallel processing
  27 #
  28 import os, re
  29 from Scientific import MPI
  30 communicator = MPI.world.duplicate()
  31
  32 #
  33 #----------- Command line arguments parser
  34 #
  35 from optparse import OptionParser
  36
  37 usage = "%prog [options] \"process\""
  38 parser = OptionParser(usage)
  39 parser.add_option("-a", "--arg_n",
  40                   action="store_true", dest="arg_n", default=False,
  41                   help="if the process needs the current number in its \
  42                   argument list instead of prepending/appending it to the \
  43                   filebase name, the number will be put before (-f) \
  44                   \"filename\" in the \"process\" string.")
  45 parser.add_option("-b", "--basename", type='string', dest="basename",
  46                   help="File basename for reading", metavar="FILE")
  47 parser.add_option("-e", "--ext", type='string', dest="ext", metavar="EXT",
  48                   help="add an extension after the file basename + number (without leading \".\")")
  49 parser.add_option("-f", "--first", type='int', dest="first_nr", default=0,
  50                   help="first numbered file (default: 0)", metavar="N")
  51 parser.add_option("-l", "--last", type='int', dest="last_nr", default=0,
  52                   help="last numbered file(default: 0)", metavar="N")
  53 parser.add_option("-i", "--incr", type='int', dest="incr_nr", default=1,
  54                   help="increment file number (default: 1)", metavar="N")
  55 parser.add_option("-p", "--print",
  56                   action="store_true", dest="pri", default=False,
  57                   help="prints process parameters/variables to stdout")
  58 parser.add_option("--pad", type='int', dest="pad0", default=0,
  59                   help="padding number with zero's (default: 0)", metavar="N")
  60 parser.add_option("-n", "--none",
  61                   action="store_true", dest="none", default=False,
  62                   help="suppresses real execution")
  63 parser.add_option("-x", "--prefix", action="store_true", dest="prefix", default=False,
  64                   help="prefix numbering to file basename")
  65
  66 (options, args) = parser.parse_args()
  67 if len(args) != 1:
  68         parser.error("incorrect number of arguments")
  69 else:
  70         process = args[0]
  71
  72
  73 #
  74 #----------- Function definitions
  75 #
  76 def pri_date(msg = "Time stamp at start of series processing:"):
  77         """Prints time stamp.
  78
  79         Keyword arguments:
  80         msg -- message to be printed before time stamp
  81         """
  82         if options.pri == True:
  83                 print msg
  84                 os.system('date')
  85         elif options.none:
  86                 print msg
  87                 os.system('date')
  88
  89
  90
  91 def count_digits(nr):
  92         """Counts number of digits from a number
  93
  94         Keyword arguments:
  95         nr -- number to be questioned
  96         """
  97         count=0
  98         while nr/10 !=0:
  99                 nr=nr/10
 100                 count=count+1
 101         return count
 102
 103
 104 def pad0(nr):
 105         """Created a string for zero padding
 106
 107         Keyword arguments:
 108         nr -- number of zeros to be padded
 109         """
 110         pd0=""
 111         for i in range(0, nr):
 112                 pd0 = str(pd0)+"0"
 113
 114         return pd0
 115
 116
 117 def compose_name_nr(nr):
 118         """Creates proper name from basename and number.
 119
 120         Keyword arguments:
 121         nr -- number of filename to be processed
 122         """
 123         if options.pad0 > 0:
 124                 ndig=count_digits(nr)
 125                 null_str=pad0(options.pad0 - ndig)
 126                 nr_str=null_str+str(nr)
 127         else:
 128                 nr_str=str(nr)
 129
 130         if options.prefix:
 131                 if options.arg_n:
 132                         name=str(options.basename)
 133                 else:
 134                         name=nr_str+str(options.basename)
 135         else:
 136                 if options.arg_n:
 137                         name=str(options.basename)
 138                 else:
 139                         name=str(options.basename)+nr_str
 140
 141         if str(options.ext) != "None":
 142                 name=str(name)+str(".")+str(options.ext)
 143
 144         return(name)
 145
 146
 147 def compose_cmd(name, nr):
 148         """Creates proper command.
 149
 150         Keyword arguments:
 151         name -- complete filename
 152         """
 153         command=str(process)
 154         if options.arg_n:
 155                 # Eventually, substitutes "-f" with: "nr -f"
 156                 command_tmp=re.sub("-f", str(nr)+" -f", command)
 157                 if command_tmp != command:
 158                         command = str(command_tmp)+" "+str(name)
 159                 else:
 160                         command = str(command_tmp)+" "+str(nr)+" "+str(name)
 161         else:
 162                 command=str(command)+" "+str(name)
 163
 164         return command
 165
 166
 167 def proc_series_par():
 168         """Processes a series on identic numbered files in parallel
 169         environment using mpipython.
 170
 171         """
 172         #
 173         # Total number of data and number of data per node
 174         #
 175         Nt = (options.last_nr+1 - options.first_nr) / options.incr_nr
 176         Nn = Nt / communicator.size
 177
 178         #
 179         # Bail out if number of nodes is larger than number of data
 180         #
 181         if communicator.size > Nt:
 182                 print 'nprocs (',communicator.size,\
 183                       ') larger then number of data (',Nt,")"
 184                 exit()
 185
 186
 187         for i in range(options.first_nr, options.last_nr+1, options.incr_nr):
 188                 for j in range(0, communicator.size, 1):
 189                         if communicator.rank == j:
 190                                 if i >= options.first_nr + j*Nn*options.incr_nr:
 191                                         if i < options.first_nr + (j+1)*Nn*options.incr_nr:
 192                                                 name_nr = compose_name_nr(i)
 193                                                 command = compose_cmd(name_nr, i)
 194                                                 if options.pri == True: print "rank=",communicator.rank,command
 195                                                 elif options.none == True: print "rank=",communicator.rank,command
 196                                                 if options.none == False: os.system(command)
 197
 198                                 #
 199                                 # The fraction of data left, if Nn is not an integer,
 200                                 # is processed at the highest node
 201                                 #
 202                                 if j == communicator.size - 1:
 203                                         if i >=  options.first_nr + (j+1)*Nn*options.incr_nr:
 204                                                 name_nr = compose_name_nr(i)
 205                                                 command = compose_cmd(name_nr, i)
 206                                                 if options.pri == True: print "rank=",communicator.rank,command
 207                                                 elif options.none == True: print "rank=",communicator.rank,command
 208                                                 if options.none == False: os.system(command)
 209
 210
 211 #
 212 #----------- Calling functions
 213 #
 214 #if communicator.rank == 0:
 215 #       if options.pri == True: pri_date()
 216 #       elif options.none == True: pri_date()
 217
 218 proc_series_par()
 219
 220 #communicator.rank == 0:
 221 #       if options.pri == True: pri_date(msg = "Time stamp at end of series processing:")
 222 #       elif options.none == True: pri_date(msg = "Time stamp at end of series processing:")
 223
 224 #
 225 #----------- That's all folks
 226 #