plugins/togo/togo.py

   1 import cookielib
   2 import logging
   3 import os
   4 import subprocess
   5 import thread
   6 import time
   7 import urllib2
   8 import urlparse
   9 from urllib import quote, unquote
  10 from xml.dom import minidom
  11 from xml.sax.saxutils import escape
  12
  13 from Cheetah.Template import Template
  14
  15 import config
  16 import metadata
  17 from plugin import EncodeUnicode, Plugin
  18
  19 logger = logging.getLogger('pyTivo.togo')
  20 tag_data = metadata.tag_data
  21
  22 SCRIPTDIR = os.path.dirname(__file__)
  23
  24 CLASS_NAME = 'ToGo'
  25
  26 # Characters to remove from filenames
  27
  28 BADCHAR = {'\\': '-', '/': '-', ':': ' -', ';': ',', '*': '.',
  29            '?': '.', '!': '.', '"': "'", '<': '(', '>': ')', '|': ' '}
  30
  31 # Some error/status message templates
  32
  33 MISSING = """<h3>Missing Data</h3> <p>You must set both "tivo_mak" and
  34 "togo_path" before using this function.</p>"""
  35
  36 TRANS_QUEUE = """<h3>Queued for Transfer</h3> <p>%s</p> <p>queued for
  37 transfer to:</p> <p>%s</p>"""
  38
  39 TRANS_STOP = """<h3>Transfer Stopped</h3> <p>Your transfer of:</p>
  40 <p>%s</p> <p>has been stopped.</p>"""
  41
  42 UNQUEUE = """<h3>Removed from Queue</h3> <p>%s</p> <p>has been removed
  43 from the queue.</p>"""
  44
  45 UNABLE = """<h3>Unable to Connect to TiVo</h3> <p>pyTivo was unable to
  46 connect to the TiVo at %s.</p> <p>This is most likely caused by an
  47 incorrect Media Access Key. Please return to the Settings page and
  48 double check your <b>tivo_mak</b> setting.</p>"""
  49
  50 # Preload the templates
  51 tnname = os.path.join(SCRIPTDIR, 'templates', 'npl.tmpl')
  52 NPL_TEMPLATE = file(tnname, 'rb').read()
  53
  54 status = {} # Global variable to control download threads
  55 tivo_cache = {} # Cache of TiVo NPL
  56 queue = {} # Recordings to download -- list per TiVo
  57 basic_meta = {} # Data from NPL, parsed, indexed by progam URL
  58
  59 def null_cookie(name, value):
  60     return cookielib.Cookie(0, name, value, None, False, '', False,
  61         False, '', False, False, None, False, None, None, None)
  62
  63 auth_handler = urllib2.HTTPDigestAuthHandler()
  64 cj = cookielib.CookieJar()
  65 cj.set_cookie(null_cookie('sid', 'ADEADDA7EDEBAC1E'))
  66 tivo_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj),
  67                                    auth_handler)
  68
  69 class ToGo(Plugin):
  70     CONTENT_TYPE = 'text/html'
  71
  72     def tivo_open(self, url):
  73         # Loop just in case we get a server busy message
  74         while True:
  75             try:
  76                 # Open the URL using our authentication/cookie opener
  77                 return tivo_opener.open(url)
  78
  79             # Do a retry if the TiVo responds that the server is busy
  80             except urllib2.HTTPError, e:
  81                 if e.code == 503:
  82                     time.sleep(5)
  83                     continue
  84
  85                 # Throw the error otherwise
  86                 raise
  87
  88     def NPL(self, handler, query):
  89
  90         def getint(thing):
  91             try:
  92                 result = int(thing)
  93             except:
  94                 result = 0
  95             return result
  96
  97         global basic_meta
  98         shows_per_page = 50 # Change this to alter the number of shows returned
  99         folder = ''
 100         FirstAnchor = ''
 101         has_tivodecode = bool(config.get_bin('tivodecode'))
 102
 103         if 'TiVo' in query:
 104             tivoIP = query['TiVo'][0]
 105             tsn = config.tivos_by_ip(tivoIP)
 106             tivo_name = config.tivo_names[tsn]
 107             tivo_mak = config.get_tsn('tivo_mak', tsn)
 108             theurl = ('https://' + tivoIP +
 109                       '/TiVoConnect?Command=QueryContainer&ItemCount=' +
 110                       str(shows_per_page) + '&Container=/NowPlaying')
 111             if 'Folder' in query:
 112                 folder += query['Folder'][0]
 113                 theurl += '/' + folder
 114             if 'AnchorItem' in query:
 115                 theurl += '&AnchorItem=' + quote(query['AnchorItem'][0])
 116             if 'AnchorOffset' in query:
 117                 theurl += '&AnchorOffset=' + query['AnchorOffset'][0]
 118
 119             if (theurl not in tivo_cache or
 120                 (time.time() - tivo_cache[theurl]['thepage_time']) >= 60):
 121                 # if page is not cached or old then retreive it
 122                 auth_handler.add_password('TiVo DVR', tivoIP, 'tivo', tivo_mak)
 123                 try:
 124                     page = self.tivo_open(theurl)
 125                 except IOError, e:
 126                     handler.redir(UNABLE % tivoIP, 10)
 127                     return
 128                 tivo_cache[theurl] = {'thepage': minidom.parse(page),
 129                                       'thepage_time': time.time()}
 130                 page.close()
 131
 132             xmldoc = tivo_cache[theurl]['thepage']
 133             items = xmldoc.getElementsByTagName('Item')
 134             TotalItems = tag_data(xmldoc, 'TiVoContainer/Details/TotalItems')
 135             ItemStart = tag_data(xmldoc, 'TiVoContainer/ItemStart')
 136             ItemCount = tag_data(xmldoc, 'TiVoContainer/ItemCount')
 137             title = tag_data(xmldoc, 'TiVoContainer/Details/Title')
 138             if items:
 139                 FirstAnchor = tag_data(items[0], 'Links/Content/Url')
 140
 141             data = []
 142             for item in items:
 143                 entry = {}
 144                 entry['ContentType'] = tag_data(item, 'Details/ContentType')
 145                 for tag in ('CopyProtected', 'UniqueId'):
 146                     value = tag_data(item, 'Details/' + tag)
 147                     if value:
 148                         entry[tag] = value
 149                 if entry['ContentType'] == 'x-tivo-container/folder':
 150                     entry['Title'] = tag_data(item, 'Details/Title')
 151                     entry['TotalItems'] = tag_data(item, 'Details/TotalItems')
 152                     lc = tag_data(item, 'Details/LastCaptureDate')
 153                     if not lc:
 154                         lc = tag_data(item, 'Details/LastChangeDate')
 155                     entry['LastChangeDate'] = time.strftime('%b %d, %Y',
 156                         time.localtime(int(lc, 16)))
 157                 else:
 158                     keys = {'Icon': 'Links/CustomIcon/Url',
 159                             'Url': 'Links/Content/Url',
 160                             'SourceSize': 'Details/SourceSize',
 161                             'Duration': 'Details/Duration',
 162                             'CaptureDate': 'Details/CaptureDate'}
 163                     for key in keys:
 164                         value = tag_data(item, keys[key])
 165                         if value:
 166                             entry[key] = value
 167
 168                     rawsize = entry['SourceSize']
 169                     entry['SourceSize'] = metadata.human_size(rawsize)
 170
 171                     dur = getint(entry['Duration']) / 1000
 172                     entry['Duration'] = ( '%d:%02d:%02d' %
 173                         (dur / 3600, (dur % 3600) / 60, dur % 60) )
 174
 175                     entry['CaptureDate'] = time.strftime('%b %d, %Y',
 176                         time.localtime(int(entry['CaptureDate'], 16)))
 177
 178                     url = entry['Url']
 179                     if url in basic_meta:
 180                         entry.update(basic_meta[url])
 181                     else:
 182                         basic_data = metadata.from_container(item)
 183                         entry.update(basic_data)
 184                         basic_meta[url] = basic_data
 185
 186                 data.append(entry)
 187         else:
 188             data = []
 189             tivoIP = ''
 190             TotalItems = 0
 191             ItemStart = 0
 192             ItemCount = 0
 193             title = ''
 194
 195         t = Template(NPL_TEMPLATE, filter=EncodeUnicode)
 196         t.escape = escape
 197         t.quote = quote
 198         t.folder = folder
 199         t.status = status
 200         if tivoIP in queue:
 201             t.queue = queue[tivoIP]
 202         t.has_tivodecode = has_tivodecode
 203         t.togo_mpegts = config.is_ts_capable(tsn)
 204         t.tname = tivo_name
 205         t.tivoIP = tivoIP
 206         t.container = handler.cname
 207         t.data = data
 208         t.len = len
 209         t.TotalItems = getint(TotalItems)
 210         t.ItemStart = getint(ItemStart)
 211         t.ItemCount = getint(ItemCount)
 212         t.FirstAnchor = quote(FirstAnchor)
 213         t.shows_per_page = shows_per_page
 214         t.title = title
 215         handler.send_html(str(t), refresh='300')
 216
 217     def get_tivo_file(self, tivoIP, url, mak, togo_path):
 218         # global status
 219         status[url].update({'running': True, 'queued': False})
 220
 221         parse_url = urlparse.urlparse(url)
 222
 223         name = unquote(parse_url[2])[10:].split('.')
 224         id = unquote(parse_url[4]).split('id=')[1]
 225         ts = status[url]['ts_format']
 226         name.insert(-1, ' - ' + id)
 227         if status[url]['decode']:
 228             if ts:
 229                 name[-1] = 'ts'
 230             else:
 231                 name[-1] = 'mpg'
 232         else:
 233             if ts:
 234                 name.insert(-1, ' (TS)')
 235             else:
 236                 name.insert(-1, ' (PS)')
 237         name.insert(-1, '.')
 238         name = ''.join(name)
 239         for ch in BADCHAR:
 240             name = name.replace(ch, BADCHAR[ch])
 241         outfile = os.path.join(togo_path, name)
 242
 243         if status[url]['save']:
 244             meta = basic_meta[url]
 245             details_url = 'https://%s/TiVoVideoDetails?id=%s' % (tivoIP, id)
 246             try:
 247                 handle = self.tivo_open(details_url)
 248                 meta.update(metadata.from_details(handle))
 249                 handle.close()
 250             except:
 251                 pass
 252             metafile = open(outfile + '.txt', 'w')
 253             metadata.dump(metafile, meta)
 254             metafile.close()
 255
 256         auth_handler.add_password('TiVo DVR', url, 'tivo', mak)
 257         try:
 258             if status[url]['ts_format']:
 259                 handle = self.tivo_open(url + '&Format=video/x-tivo-mpeg-ts')
 260             else:
 261                 handle = self.tivo_open(url)
 262         except urllib2.HTTPError, e:
 263             status[url]['running'] = False
 264             status[url]['error'] = e.code
 265             logger.error(e.code)
 266             return
 267         except urllib2.URLError, e:
 268             status[url]['running'] = False
 269             status[url]['error'] = e.reason
 270             logger.error(e.reason)
 271             return
 272
 273         tivo_name = config.tivo_names[config.tivos_by_ip(tivoIP)]
 274
 275         logger.info('[%s] Start getting "%s" from %s' %
 276                     (time.strftime('%d/%b/%Y %H:%M:%S'), outfile, tivo_name))
 277
 278         if status[url]['decode']:
 279             tivodecode_path = config.get_bin('tivodecode')
 280             tcmd = [tivodecode_path, '-m', mak, '-o', outfile, '-']
 281             tivodecode = subprocess.Popen(tcmd, stdin=subprocess.PIPE,
 282                                           bufsize=(512 * 1024))
 283             f = tivodecode.stdin
 284         else:
 285             f = open(outfile, 'wb')
 286         length = 0
 287         start_time = time.time()
 288         last_interval = start_time
 289         now = start_time
 290         try:
 291             while status[url]['running']:
 292                 output = handle.read(1024000)
 293                 if not output:
 294                     break
 295                 length += len(output)
 296                 f.write(output)
 297                 now = time.time()
 298                 elapsed = now - last_interval
 299                 if elapsed >= 5:
 300                     status[url]['rate'] = '%.2f Mb/s' % (length * 8.0 /
 301                         (elapsed * 1024 * 1024))
 302                     status[url]['size'] += length
 303                     length = 0
 304                     last_interval = now
 305             if status[url]['running']:
 306                 status[url]['finished'] = True
 307         except Exception, msg:
 308             status[url]['running'] = False
 309             logger.info(msg)
 310         handle.close()
 311         f.close()
 312         status[url]['size'] += length
 313         if status[url]['running']:
 314             mega_elapsed = (now - start_time) * 1024 * 1024
 315             if mega_elapsed < 1:
 316                 mega_elapsed = 1
 317             size = status[url]['size']
 318             rate = size * 8.0 / mega_elapsed
 319             logger.info('[%s] Done getting "%s" from %s, %d bytes, %.2f Mb/s' %
 320                         (time.strftime('%d/%b/%Y %H:%M:%S'), outfile,
 321                          tivo_name, size, rate))
 322             status[url]['running'] = False
 323         else:
 324             os.remove(outfile)
 325             if status[url]['save']:
 326                 os.remove(outfile + '.txt')
 327             logger.info('[%s] Transfer of "%s" from %s aborted' %
 328                         (time.strftime('%d/%b/%Y %H:%M:%S'), outfile,
 329                          tivo_name))
 330             del status[url]
 331
 332     def process_queue(self, tivoIP, mak, togo_path):
 333         while queue[tivoIP]:
 334             time.sleep(5)
 335             url = queue[tivoIP][0]
 336             self.get_tivo_file(tivoIP, url, mak, togo_path)
 337             queue[tivoIP].pop(0)
 338         del queue[tivoIP]
 339
 340     def ToGo(self, handler, query):
 341         togo_path = config.get_server('togo_path')
 342         for name, data in config.getShares():
 343             if togo_path == name:
 344                 togo_path = data.get('path')
 345         if togo_path:
 346             tivoIP = query['TiVo'][0]
 347             tsn = config.tivos_by_ip(tivoIP)
 348             tivo_mak = config.get_tsn('tivo_mak', tsn)
 349             urls = query.get('Url', [])
 350             decode = 'decode' in query
 351             save = 'save' in query
 352             ts_format = 'ts_format' in query
 353             for theurl in urls:
 354                 status[theurl] = {'running': False, 'error': '', 'rate': '',
 355                                   'queued': True, 'size': 0, 'finished': False,
 356                                   'decode': decode, 'save': save,
 357                                   'ts_format': ts_format}
 358                 if tivoIP in queue:
 359                     queue[tivoIP].append(theurl)
 360                 else:
 361                     queue[tivoIP] = [theurl]
 362                     thread.start_new_thread(ToGo.process_queue,
 363                                             (self, tivoIP, tivo_mak, togo_path))
 364                 logger.info('[%s] Queued "%s" for transfer to %s' %
 365                             (time.strftime('%d/%b/%Y %H:%M:%S'),
 366                              unquote(theurl), togo_path))
 367             urlstring = '<br>'.join([unquote(x) for x in urls])
 368             message = TRANS_QUEUE % (urlstring, togo_path)
 369         else:
 370             message = MISSING
 371         handler.redir(message, 5)
 372
 373     def ToGoStop(self, handler, query):
 374         theurl = query['Url'][0]
 375         status[theurl]['running'] = False
 376         handler.redir(TRANS_STOP % unquote(theurl))
 377
 378     def Unqueue(self, handler, query):
 379         theurl = query['Url'][0]
 380         tivoIP = query['TiVo'][0]
 381         del status[theurl]
 382         queue[tivoIP].remove(theurl)
 383         logger.info('[%s] Removed "%s" from queue' %
 384                     (time.strftime('%d/%b/%Y %H:%M:%S'),
 385                      unquote(theurl)))
 386         handler.redir(UNQUEUE % unquote(theurl))