pdb-gtl/make_merged_set.py

   1 #!/usr/bin/python
   2
   3 import os, random, shutil
   4 raw= 'rawpat_files/'
   5 out= 'rawpat_files_merged/'
   6 ranks = os.listdir(raw)
   7 limit = 400000 # bytes
   8
   9 for rank in ranks:
  10         merge = [[]]
  11         last_size = 0
  12         files = os.listdir(raw + rank)
  13         for f in files:
  14                 size = os.stat(raw + rank + '/' + f).st_size
  15                 last_size += size
  16                 merge[-1] += [f]
  17                 if last_size >= limit:
  18                         last_size = 0
  19                         merge += [[]]
  20         if len(merge) > 1:
  21                 merge[-2] += merge[-1]
  22                 del merge[-1]
  23
  24         try:
  25                 os.makedirs(out + rank)
  26         except:
  27                 pass
  28
  29         #print merge
  30         for num, fs in enumerate(merge):
  31                 outf = open( out + rank + '/' + str(num), 'wb' )
  32                 for f in fs:
  33                         shutil.copyfileobj(open(raw + rank + '/' + f, 'rb'), outf)
  34                 outf.close()