retrieve_perims.py

   1 import glob, re, sys, os, pytz, requests
   2 from JPSSD import time_iso2datetime, time_datetime2iso, get_url
   3 from utils import clean_dir
   4 import os.path as osp
   5 from datetime import datetime,timedelta
   6 from tzwhere import tzwhere
   7
   8 if len(sys.argv) != 3:
   9     print 'Error: python %s firename year' % sys.argv[0]
  10     sys.exit(1)
  11
  12 firename = sys.argv[1]
  13 year = sys.argv[2]
  14 if year == str(datetime.today().year):
  15     year = 'current_year'
  16 dst_in = 'perim_orig'
  17 dst_out = 'perim'
  18
  19 clean_dir(dst_in)
  20 clean_dir(dst_out)
  21 baseurl = 'https://rmgsc.cr.usgs.gov/outgoing/GeoMAC/'
  22 url = osp.join(baseurl,year+'_fire_data/KMLS/')
  23 r = requests.get(url, stream=True)
  24 content = r.content
  25 plist = re.findall('([a-z\d\s-]+%s[\d\s-]+.kml)' % firename,content,re.IGNORECASE)
  26 for p in plist:
  27     get_url(osp.join(url,p),osp.join(dst_in,p))
  28
  29 files = glob.glob(osp.join(dst_in, '*.kml'))
  30 print 'Transforming KML files to UTC from %s to %s' % (dst_in, dst_out)
  31 for k,file in enumerate(files):
  32     f = open(file,"r")
  33     f_str = ''.join(f.readlines())
  34     f.close()
  35     name = re.findall(r'<name>(.*?)</name>',f_str,re.DOTALL)[0]
  36     match = re.match(r'(.*) ([0-9]+)-([0-9]+)-([0-9]+) ([0-9]{2})([0-9]{2})',name).groups()
  37     case = match[0]
  38     date = (match[3],match[1],match[2],match[4],match[5])
  39     time_iso = '%04d-%02d-%02dT%02d:%02d:00Z' % tuple([ int(d) for d in date ])
  40     time_datetime = time_iso2datetime(time_iso)
  41     # Calculate time zone from lon/lat in the first file
  42     if not k:
  43         print 'Computing GMT from coordinates in the first file...'
  44         coord = re.findall(r'<coordinates>(.*?)</coordinates>',f_str,re.DOTALL)[0]
  45         lon,lat,_ = map(float,re.findall(r'([-]?[0-9.]+)',coord,re.DOTALL))
  46         tz = tzwhere.tzwhere(forceTZ=True)
  47         timezone_str = tz.tzNameAt(lat,lon,forceTZ = True)
  48         timezone = pytz.timezone(timezone_str)
  49         gmt = timezone.utcoffset(time_datetime).total_seconds()/3600.
  50         print 'GMT%d' % gmt
  51     new_time_datetime = time_datetime + timedelta(hours=-gmt)
  52     new_time_iso = time_datetime2iso(new_time_datetime)
  53     new_name = case + ' %02d-%02d-%04d %02d%02d' % (new_time_datetime.month,
  54                                                     new_time_datetime.day,
  55                                                     new_time_datetime.year,
  56                                                     new_time_datetime.hour,
  57                                                     new_time_datetime.minute)
  58     new_file = osp.join(dst_out,new_name+'.kml')
  59     new_f = open(new_file,"w")
  60     new_f_str = re.sub(r'(?is)<name>(.*?)</name>(?is)', '<name>'+new_name+'</name>', f_str)
  61     new_f_str = re.sub(r'(?is)</Placemark>(?is)','<TimeStamp><when>'+new_time_iso+'</when></TimeStamp>\n</Placemark>',new_f_str)
  62     # not always same structure: be careful to loose information
  63     new_f_str = re.sub(r'(?is)<Placemark>(.*?)</Placemark>(?is)','',new_f_str,1)
  64     new_f.write(new_f_str)
  65     new_f.close()
  66     print '> new file %s created.' % new_file