plugins/togo/togo.py

   1 import cookielib
   2 import logging
   3 import os
   4 import subprocess
   5 import thread
   6 import time
   7 import urllib2
   8 import urlparse
   9 from urllib import quote, unquote
  10 from xml.dom import minidom
  11 from xml.sax.saxutils import escape
  12
  13 from Cheetah.Template import Template
  14
  15 import config
  16 import metadata
  17 from plugin import EncodeUnicode, Plugin
  18
  19 logger = logging.getLogger('pyTivo.togo')
  20 tag_data = metadata.tag_data
  21
  22 SCRIPTDIR = os.path.dirname(__file__)
  23
  24 CLASS_NAME = 'ToGo'
  25
  26 # Characters to remove from filenames
  27
  28 BADCHAR = {'\\': '-', '/': '-', ':': ' -', ';': ',', '*': '.',
  29            '?': '.', '!': '.', '"': "'", '<': '(', '>': ')', '|': ' '}
  30
  31 # Some error/status message templates
  32
  33 MISSING = """<h3>Missing Data</h3> <p>You must set both "tivo_mak" and
  34 "togo_path" before using this function.</p>"""
  35
  36 TRANS_QUEUE = """<h3>Queued for Transfer</h3> <p>%s</p> <p>queued for
  37 transfer to:</p> <p>%s</p>"""
  38
  39 TRANS_STOP = """<h3>Transfer Stopped</h3> <p>Your transfer of:</p>
  40 <p>%s</p> <p>has been stopped.</p>"""
  41
  42 UNQUEUE = """<h3>Removed from Queue</h3> <p>%s</p> <p>has been removed
  43 from the queue.</p>"""
  44
  45 UNABLE = """<h3>Unable to Connect to TiVo</h3> <p>pyTivo was unable to
  46 connect to the TiVo at %s.</p> <p>This is most likely caused by an
  47 incorrect Media Access Key. Please return to the Settings page and
  48 double check your <b>tivo_mak</b> setting.</p> <pre>%s</pre>"""
  49
  50 # Preload the templates
  51 def tmpl(name):
  52     return file(os.path.join(SCRIPTDIR, 'templates', name), 'rb').read()
  53
  54 CONTAINER_TEMPLATE_MOBILE = tmpl('npl_mob.tmpl')
  55 CONTAINER_TEMPLATE = tmpl('npl.tmpl')
  56
  57 status = {} # Global variable to control download threads
  58 tivo_cache = {} # Cache of TiVo NPL
  59 queue = {} # Recordings to download -- list per TiVo
  60 basic_meta = {} # Data from NPL, parsed, indexed by progam URL
  61
  62 def null_cookie(name, value):
  63     return cookielib.Cookie(0, name, value, None, False, '', False,
  64         False, '', False, False, None, False, None, None, None)
  65
  66 auth_handler = urllib2.HTTPDigestAuthHandler()
  67 cj = cookielib.CookieJar()
  68 cj.set_cookie(null_cookie('sid', 'ADEADDA7EDEBAC1E'))
  69 tivo_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj),
  70                                    auth_handler)
  71
  72 class ToGo(Plugin):
  73     CONTENT_TYPE = 'text/html'
  74
  75     def tivo_open(self, url):
  76         # Loop just in case we get a server busy message
  77         while True:
  78             try:
  79                 # Open the URL using our authentication/cookie opener
  80                 return tivo_opener.open(url)
  81
  82             # Do a retry if the TiVo responds that the server is busy
  83             except urllib2.HTTPError, e:
  84                 if e.code == 503:
  85                     time.sleep(5)
  86                     continue
  87
  88                 # Log and throw the error otherwise
  89                 logger.error(e)
  90                 raise
  91
  92     def NPL(self, handler, query):
  93
  94         def getint(thing):
  95             try:
  96                 result = int(thing)
  97             except:
  98                 result = 0
  99             return result
 100
 101         global basic_meta
 102         shows_per_page = 50 # Change this to alter the number of shows returned
 103         folder = ''
 104         FirstAnchor = ''
 105         has_tivodecode = bool(config.get_bin('tivodecode'))
 106
 107         if 'TiVo' in query:
 108             tivoIP = query['TiVo'][0]
 109             tsn = config.tivos_by_ip(tivoIP)
 110             useragent = handler.headers.getheader('User-Agent', '')
 111
 112             tivo_name = config.tivo_names[tsn]
 113             tivo_mak = config.get_tsn('tivo_mak', tsn)
 114             theurl = ('https://' + tivoIP +
 115                       '/TiVoConnect?Command=QueryContainer&ItemCount=' +
 116                       str(shows_per_page) + '&Container=/NowPlaying')
 117             if 'Folder' in query:
 118                 folder += query['Folder'][0]
 119                 theurl += '/' + folder
 120             if 'AnchorItem' in query:
 121                 theurl += '&AnchorItem=' + quote(query['AnchorItem'][0])
 122             if 'AnchorOffset' in query:
 123                 theurl += '&AnchorOffset=' + query['AnchorOffset'][0]
 124
 125             if (theurl not in tivo_cache or
 126                 (time.time() - tivo_cache[theurl]['thepage_time']) >= 60):
 127                 # if page is not cached or old then retreive it
 128                 auth_handler.add_password('TiVo DVR', tivoIP, 'tivo', tivo_mak)
 129                 try:
 130                     page = self.tivo_open(theurl)
 131                 except IOError, e:
 132                     handler.redir(UNABLE % (tivoIP, e), 10)
 133                     return
 134                 tivo_cache[theurl] = {'thepage': minidom.parse(page),
 135                                       'thepage_time': time.time()}
 136                 page.close()
 137
 138             xmldoc = tivo_cache[theurl]['thepage']
 139             items = xmldoc.getElementsByTagName('Item')
 140             TotalItems = tag_data(xmldoc, 'TiVoContainer/Details/TotalItems')
 141             ItemStart = tag_data(xmldoc, 'TiVoContainer/ItemStart')
 142             ItemCount = tag_data(xmldoc, 'TiVoContainer/ItemCount')
 143             title = tag_data(xmldoc, 'TiVoContainer/Details/Title')
 144             if items:
 145                 FirstAnchor = tag_data(items[0], 'Links/Content/Url')
 146
 147             data = []
 148             for item in items:
 149                 entry = {}
 150                 entry['ContentType'] = tag_data(item, 'Details/ContentType')
 151                 for tag in ('CopyProtected', 'UniqueId'):
 152                     value = tag_data(item, 'Details/' + tag)
 153                     if value:
 154                         entry[tag] = value
 155                 if entry['ContentType'] == 'x-tivo-container/folder':
 156                     entry['Title'] = tag_data(item, 'Details/Title')
 157                     entry['TotalItems'] = tag_data(item, 'Details/TotalItems')
 158                     lc = tag_data(item, 'Details/LastCaptureDate')
 159                     if not lc:
 160                         lc = tag_data(item, 'Details/LastChangeDate')
 161                     entry['LastChangeDate'] = time.strftime('%b %d, %Y',
 162                         time.localtime(int(lc, 16)))
 163                 else:
 164                     keys = {'Icon': 'Links/CustomIcon/Url',
 165                             'Url': 'Links/Content/Url',
 166                             'SourceSize': 'Details/SourceSize',
 167                             'Duration': 'Details/Duration',
 168                             'CaptureDate': 'Details/CaptureDate'}
 169                     for key in keys:
 170                         value = tag_data(item, keys[key])
 171                         if value:
 172                             entry[key] = value
 173
 174                     rawsize = entry['SourceSize']
 175                     entry['SourceSize'] = metadata.human_size(rawsize)
 176
 177                     dur = getint(entry['Duration']) / 1000
 178                     entry['Duration'] = ( '%d:%02d:%02d' %
 179                         (dur / 3600, (dur % 3600) / 60, dur % 60) )
 180
 181                     entry['CaptureDate'] = time.strftime('%b %d, %Y',
 182                         time.localtime(int(entry['CaptureDate'], 16)))
 183
 184                     url = entry['Url']
 185                     if url in basic_meta:
 186                         entry.update(basic_meta[url])
 187                     else:
 188                         basic_data = metadata.from_container(item)
 189                         entry.update(basic_data)
 190                         basic_meta[url] = basic_data
 191
 192                 data.append(entry)
 193         else:
 194             data = []
 195             tivoIP = ''
 196             TotalItems = 0
 197             ItemStart = 0
 198             ItemCount = 0
 199             title = ''
 200
 201         if useragent.lower().find('mobile') > 0:
 202             t = Template(CONTAINER_TEMPLATE_MOBILE, filter=EncodeUnicode)
 203         else:
 204             t = Template(CONTAINER_TEMPLATE, filter=EncodeUnicode)
 205         t.escape = escape
 206         t.quote = quote
 207         t.folder = folder
 208         t.status = status
 209         if tivoIP in queue:
 210             t.queue = queue[tivoIP]
 211         t.has_tivodecode = has_tivodecode
 212         t.togo_mpegts = config.is_ts_capable(tsn)
 213         t.tname = tivo_name
 214         t.tivoIP = tivoIP
 215         t.container = handler.cname
 216         t.data = data
 217         t.len = len
 218         t.TotalItems = getint(TotalItems)
 219         t.ItemStart = getint(ItemStart)
 220         t.ItemCount = getint(ItemCount)
 221         t.FirstAnchor = quote(FirstAnchor)
 222         t.shows_per_page = shows_per_page
 223         t.title = title
 224         handler.send_html(str(t), refresh='300')
 225
 226     def get_tivo_file(self, tivoIP, url, mak, togo_path):
 227         # global status
 228         status[url].update({'running': True, 'queued': False})
 229
 230         parse_url = urlparse.urlparse(url)
 231
 232         name = unquote(parse_url[2])[10:].split('.')
 233         id = unquote(parse_url[4]).split('id=')[1]
 234         ts = status[url]['ts_format']
 235         name.insert(-1, ' - ' + id)
 236         if status[url]['decode']:
 237             if ts:
 238                 name[-1] = 'ts'
 239             else:
 240                 name[-1] = 'mpg'
 241         else:
 242             if ts:
 243                 name.insert(-1, ' (TS)')
 244             else:
 245                 name.insert(-1, ' (PS)')
 246         name.insert(-1, '.')
 247         name = ''.join(name)
 248         for ch in BADCHAR:
 249             name = name.replace(ch, BADCHAR[ch])
 250         outfile = os.path.join(togo_path, name)
 251
 252         if status[url]['save']:
 253             meta = basic_meta[url]
 254             details_url = 'https://%s/TiVoVideoDetails?id=%s' % (tivoIP, id)
 255             try:
 256                 handle = self.tivo_open(details_url)
 257                 meta.update(metadata.from_details(handle.read()))
 258                 handle.close()
 259             except:
 260                 pass
 261             metafile = open(outfile + '.txt', 'w')
 262             metadata.dump(metafile, meta)
 263             metafile.close()
 264
 265         auth_handler.add_password('TiVo DVR', url, 'tivo', mak)
 266         try:
 267             if status[url]['ts_format']:
 268                 handle = self.tivo_open(url + '&Format=video/x-tivo-mpeg-ts')
 269             else:
 270                 handle = self.tivo_open(url)
 271         except Exception, msg:
 272             status[url]['running'] = False
 273             status[url]['error'] = str(msg)
 274             return
 275
 276         tivo_name = config.tivo_names[config.tivos_by_ip(tivoIP)]
 277
 278         logger.info('[%s] Start getting "%s" from %s' %
 279                     (time.strftime('%d/%b/%Y %H:%M:%S'), outfile, tivo_name))
 280
 281         if status[url]['decode']:
 282             tivodecode_path = config.get_bin('tivodecode')
 283             tcmd = [tivodecode_path, '-m', mak, '-o', outfile, '-']
 284             tivodecode = subprocess.Popen(tcmd, stdin=subprocess.PIPE,
 285                                           bufsize=(512 * 1024))
 286             f = tivodecode.stdin
 287         else:
 288             f = open(outfile, 'wb')
 289         length = 0
 290         start_time = time.time()
 291         last_interval = start_time
 292         now = start_time
 293         try:
 294             while status[url]['running']:
 295                 output = handle.read(1024000)
 296                 if not output:
 297                     break
 298                 length += len(output)
 299                 f.write(output)
 300                 now = time.time()
 301                 elapsed = now - last_interval
 302                 if elapsed >= 5:
 303                     status[url]['rate'] = '%.2f Mb/s' % (length * 8.0 /
 304                         (elapsed * 1024 * 1024))
 305                     status[url]['size'] += length
 306                     length = 0
 307                     last_interval = now
 308             if status[url]['running']:
 309                 status[url]['finished'] = True
 310         except Exception, msg:
 311             status[url]['running'] = False
 312             logger.info(msg)
 313         handle.close()
 314         f.close()
 315         status[url]['size'] += length
 316         if status[url]['running']:
 317             mega_elapsed = (now - start_time) * 1024 * 1024
 318             if mega_elapsed < 1:
 319                 mega_elapsed = 1
 320             size = status[url]['size']
 321             rate = size * 8.0 / mega_elapsed
 322             logger.info('[%s] Done getting "%s" from %s, %d bytes, %.2f Mb/s' %
 323                         (time.strftime('%d/%b/%Y %H:%M:%S'), outfile,
 324                          tivo_name, size, rate))
 325             status[url]['running'] = False
 326         else:
 327             os.remove(outfile)
 328             if status[url]['save']:
 329                 os.remove(outfile + '.txt')
 330             logger.info('[%s] Transfer of "%s" from %s aborted' %
 331                         (time.strftime('%d/%b/%Y %H:%M:%S'), outfile,
 332                          tivo_name))
 333             del status[url]
 334
 335     def process_queue(self, tivoIP, mak, togo_path):
 336         while queue[tivoIP]:
 337             time.sleep(5)
 338             url = queue[tivoIP][0]
 339             self.get_tivo_file(tivoIP, url, mak, togo_path)
 340             queue[tivoIP].pop(0)
 341         del queue[tivoIP]
 342
 343     def ToGo(self, handler, query):
 344         togo_path = config.get_server('togo_path')
 345         for name, data in config.getShares():
 346             if togo_path == name:
 347                 togo_path = data.get('path')
 348         if togo_path:
 349             tivoIP = query['TiVo'][0]
 350             tsn = config.tivos_by_ip(tivoIP)
 351             tivo_mak = config.get_tsn('tivo_mak', tsn)
 352             urls = query.get('Url', [])
 353             decode = 'decode' in query
 354             save = 'save' in query
 355             ts_format = 'ts_format' in query
 356             for theurl in urls:
 357                 status[theurl] = {'running': False, 'error': '', 'rate': '',
 358                                   'queued': True, 'size': 0, 'finished': False,
 359                                   'decode': decode, 'save': save,
 360                                   'ts_format': ts_format}
 361                 if tivoIP in queue:
 362                     queue[tivoIP].append(theurl)
 363                 else:
 364                     queue[tivoIP] = [theurl]
 365                     thread.start_new_thread(ToGo.process_queue,
 366                                             (self, tivoIP, tivo_mak, togo_path))
 367                 logger.info('[%s] Queued "%s" for transfer to %s' %
 368                             (time.strftime('%d/%b/%Y %H:%M:%S'),
 369                              unquote(theurl), togo_path))
 370             urlstring = '<br>'.join([unquote(x) for x in urls])
 371             message = TRANS_QUEUE % (urlstring, togo_path)
 372         else:
 373             message = MISSING
 374         handler.redir(message, 5)
 375
 376     def ToGoStop(self, handler, query):
 377         theurl = query['Url'][0]
 378         status[theurl]['running'] = False
 379         handler.redir(TRANS_STOP % unquote(theurl))
 380
 381     def Unqueue(self, handler, query):
 382         theurl = query['Url'][0]
 383         tivoIP = query['TiVo'][0]
 384         del status[theurl]
 385         queue[tivoIP].remove(theurl)
 386         logger.info('[%s] Removed "%s" from queue' %
 387                     (time.strftime('%d/%b/%Y %H:%M:%S'),
 388                      unquote(theurl)))
 389         handler.redir(UNQUEUE % unquote(theurl))