src/misc/series_mpi.py

   1 #!/usr/bin/env mpipython
   2
   3
   4 #   gpiv_series - Processes a set of numbered input data
   5
   6 #   Copyright (C) 2008 Gerber van der Graaf
   7
   8 #   This program is free software; you can redistribute it and/or modify
   9 #   it under the terms of the GNU General Public License as published by
  10 #   the Free Software Foundation; either version 2, or (at your option)
  11 #   any later version.
  12
  13 #   This program is distributed in the hope that it will be useful,
  14 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 #   GNU General Public License for more details.
  17
  18 #   You should have received a copy of the GNU General Public License
  19 #   along with this program; if not, write to the Free Software Foundation,
  20 #   Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  21
  22
  23 #--------------------------------------------------------------------
  24
  25 #
  26 # This version is MPI enabled for parallel processing
  27 #
  28 import os, re
  29 from Scientific import MPI
  30 communicator = MPI.world.duplicate()
  31
  32 #
  33 #----------- Command line arguments parser
  34 #
  35 from optparse import OptionParser
  36
  37 usage = "%prog [options] \"process\""
  38 parser = OptionParser(usage)
  39 parser.add_option("-a", "--arg_n",
  40                   action="store_true", dest="arg_n", default=False,
  41                   help="if the process needs the current number in its \
  42                   argument list instead of prepending/appending it to the \
  43                   filebase name, the number will be put before (-f) \
  44                   \"filename\" in the \"process\" string.")
  45 parser.add_option("-b", "--basename", type='string', dest="basename",
  46                   help="File basename for reading", metavar="FILE")
  47 parser.add_option("-e", "--ext", type='string', dest="ext", metavar="EXT",
  48                   help="add an extension after the file basename + number (without leading \".\")")
  49 parser.add_option("-f", "--first", type='int', dest="first_nr", default=0,
  50                   help="first numbered file (default: 0)", metavar="N")
  51 parser.add_option("-l", "--last", type='int', dest="last_nr", default=0,
  52                   help="last numbered file(default: 0)", metavar="N")
  53 parser.add_option("-i", "--incr", type='int', dest="incr_nr", default=1,
  54                   help="increment file number (default: 1)", metavar="N")
  55 parser.add_option("-p", "--print",
  56                   action="store_true", dest="pri", default=False,
  57                   help="prints process parameters/variables to stdout")
  58 parser.add_option("--pad", type='int', dest="pad0", default=0,
  59                   help="padding number with zero's (default: 0)", metavar="N")
  60 parser.add_option("-s", "--subst_fname",
  61                   action="store_true", dest="subst_fname", default=False,
  62                   help="substitutes all occurences \"FNAME\" in the \"process\" string by the actual \
  63 file basename. The option --arg_n will not have effect")
  64 parser.add_option("-n", "--none",
  65                   action="store_true", dest="none", default=False,
  66                   help="suppresses real execution")
  67 parser.add_option("-x", "--prefix", action="store_true", dest="prefix", default=False,
  68                   help="prefix numbering to file basename")
  69
  70 (options, args) = parser.parse_args()
  71 if len(args) != 1:
  72         parser.error("incorrect number of arguments")
  73 else:
  74         process = args[0]
  75
  76
  77 #
  78 #----------- Function definitions
  79 #
  80 def pri_date(msg = "Time stamp at start of series processing:"):
  81         """Prints time stamp.
  82
  83         Keyword arguments:
  84         msg -- message to be printed before time stamp
  85         """
  86         if options.pri == True:
  87                 print msg
  88                 os.system('date')
  89         elif options.none:
  90                 print msg
  91                 os.system('date')
  92
  93
  94
  95 def count_digits(nr):
  96         """Counts number of digits from a number
  97
  98         Keyword arguments:
  99         nr -- number to be questioned
 100         """
 101         count=0
 102         while nr/10 !=0:
 103                 nr=nr/10
 104                 count=count+1
 105         return count
 106
 107
 108 def pad0(nr):
 109         """Created a string for zero padding
 110
 111         Keyword arguments:
 112         nr -- number of zeros to be padded
 113         """
 114         pd0=""
 115         for i in range(0, nr):
 116                 pd0 = str(pd0)+"0"
 117
 118         return pd0
 119
 120
 121 def compose_name_nr(nr):
 122         """Creates proper name from basename and number.
 123
 124         Keyword arguments:
 125         nr -- number of filename to be processed
 126         """
 127         if options.pad0 > 0:
 128                 ndig=count_digits(nr)
 129                 null_str=pad0(options.pad0 - ndig)
 130                 nr_str=null_str+str(nr)
 131         else:
 132                 nr_str=str(nr)
 133
 134         if options.prefix:
 135                 if options.arg_n:
 136                         name=str(options.basename)
 137                 else:
 138                         name=nr_str+str(options.basename)
 139         else:
 140                 if options.arg_n:
 141                         name=str(options.basename)
 142                 else:
 143                         name=str(options.basename)+nr_str
 144
 145         if str(options.ext) != "None":
 146                 name=str(name)+str(".")+str(options.ext)
 147
 148         return(name)
 149
 150
 151 def compose_cmd(name, nr):
 152         """Creates proper command.
 153
 154         Keyword arguments:
 155         name -- complete filename
 156         """
 157         command=str(process)
 158         # Eventually, substitutes "-f" with: "nr -f"
 159         if options.arg_n:
 160                 command_tmp=re.sub("-f", str(nr)+" -f", command)
 161                 if command_tmp != command:
 162                         command = str(command_tmp)+" "+str(name)
 163                 else:
 164                         command = str(command_tmp)+" "+str(nr)+" "+str(name)
 165         else:
 166                 # Eventually, substitutes "FNAME" with the file base name
 167                 if options.subst_fname:
 168                         command=re.sub("FNAME", str(name), str(process))
 169                 else:
 170                 command=str(command)+" "+str(name)
 171
 172         return command
 173
 174
 175 def proc_series_par():
 176         """Processes a series on identic numbered files in parallel
 177         environment using mpipython.
 178
 179         """
 180         #
 181         # Total number of data and number of data per node
 182         #
 183         Nt = (options.last_nr+1 - options.first_nr) / options.incr_nr
 184         Nn = Nt / communicator.size
 185
 186         #
 187         # Bail out if number of nodes is larger than number of data
 188         #
 189         if communicator.size > Nt:
 190                 print 'nprocs (',communicator.size,\
 191                       ') larger then number of data (',Nt,")"
 192                 exit()
 193
 194
 195         for i in range(options.first_nr, options.last_nr+1, options.incr_nr):
 196                 for j in range(0, communicator.size, 1):
 197                         if communicator.rank == j:
 198                                 if i >= options.first_nr + j*Nn*options.incr_nr:
 199                                         if i < options.first_nr + (j+1)*Nn*options.incr_nr:
 200                                                 name_nr = compose_name_nr(i)
 201                                                 command = compose_cmd(name_nr, i)
 202                                                 if options.pri == True: print "rank=",communicator.rank,command
 203                                                 elif options.none == True: print "rank=",communicator.rank,command
 204                                                 if options.none == False: os.system(command)
 205
 206                                 #
 207                                 # The fraction of data left, if Nn is not an integer,
 208                                 # is processed at the highest node
 209                                 #
 210                                 if j == communicator.size - 1:
 211                                         if i >=  options.first_nr + (j+1)*Nn*options.incr_nr:
 212                                                 name_nr = compose_name_nr(i)
 213                                                 command = compose_cmd(name_nr, i)
 214                                                 if options.pri == True: print "rank=",communicator.rank,command
 215                                                 elif options.none == True: print "rank=",communicator.rank,command
 216                                                 if options.none == False: os.system(command)
 217
 218
 219 #
 220 #----------- Calling functions
 221 #
 222 #if communicator.rank == 0:
 223 #       if options.pri == True: pri_date()
 224 #       elif options.none == True: pri_date()
 225
 226 proc_series_par()
 227
 228 #communicator.rank == 0:
 229 #       if options.pri == True: pri_date(msg = "Time stamp at end of series processing:")
 230 #       elif options.none == True: pri_date(msg = "Time stamp at end of series processing:")
 231
 232 #
 233 #----------- That's all folks
 234 #