plugins/togo/togo.py

   1 import cookielib
   2 import logging
   3 import os
   4 import subprocess
   5 import thread
   6 import time
   7 import urllib2
   8 import urlparse
   9 from urllib import quote, unquote
  10 from xml.dom import minidom
  11 from xml.sax.saxutils import escape
  12
  13 from Cheetah.Template import Template
  14
  15 import config
  16 import metadata
  17 from plugin import EncodeUnicode, Plugin
  18
  19 logger = logging.getLogger('pyTivo.togo')
  20 tag_data = metadata.tag_data
  21
  22 SCRIPTDIR = os.path.dirname(__file__)
  23
  24 CLASS_NAME = 'ToGo'
  25
  26 # Characters to remove from filenames
  27
  28 BADCHAR = {'\\': '-', '/': '-', ':': ' -', ';': ',', '*': '.',
  29            '?': '.', '!': '.', '"': "'", '<': '(', '>': ')', '|': ' '}
  30
  31 # Default top-level share path
  32
  33 DEFPATH = '/TiVoConnect?Command=QueryContainer&Container=/NowPlaying'
  34
  35 # Some error/status message templates
  36
  37 MISSING = """<h3>Missing Data</h3> <p>You must set both "tivo_mak" and
  38 "togo_path" before using this function.</p>"""
  39
  40 TRANS_QUEUE = """<h3>Queued for Transfer</h3> <p>%s</p> <p>queued for
  41 transfer to:</p> <p>%s</p>"""
  42
  43 TRANS_STOP = """<h3>Transfer Stopped</h3> <p>Your transfer of:</p>
  44 <p>%s</p> <p>has been stopped.</p>"""
  45
  46 UNQUEUE = """<h3>Removed from Queue</h3> <p>%s</p> <p>has been removed
  47 from the queue.</p>"""
  48
  49 UNABLE = """<h3>Unable to Connect to TiVo</h3> <p>pyTivo was unable to
  50 connect to the TiVo at %s.</p> <p>This is most likely caused by an
  51 incorrect Media Access Key. Please return to the Settings page and
  52 double check your <b>tivo_mak</b> setting.</p> <pre>%s</pre>"""
  53
  54 # Preload the templates
  55 def tmpl(name):
  56     return file(os.path.join(SCRIPTDIR, 'templates', name), 'rb').read()
  57
  58 CONTAINER_TEMPLATE_MOBILE = tmpl('npl_mob.tmpl')
  59 CONTAINER_TEMPLATE = tmpl('npl.tmpl')
  60
  61 status = {} # Global variable to control download threads
  62 tivo_cache = {} # Cache of TiVo NPL
  63 queue = {} # Recordings to download -- list per TiVo
  64 basic_meta = {} # Data from NPL, parsed, indexed by progam URL
  65
  66 def null_cookie(name, value):
  67     return cookielib.Cookie(0, name, value, None, False, '', False,
  68         False, '', False, False, None, False, None, None, None)
  69
  70 auth_handler = urllib2.HTTPPasswordMgrWithDefaultRealm()
  71 cj = cookielib.CookieJar()
  72 cj.set_cookie(null_cookie('sid', 'ADEADDA7EDEBAC1E'))
  73 tivo_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj),
  74                                    urllib2.HTTPBasicAuthHandler(auth_handler),
  75                                    urllib2.HTTPDigestAuthHandler(auth_handler))
  76
  77 class ToGo(Plugin):
  78     CONTENT_TYPE = 'text/html'
  79
  80     def tivo_open(self, url):
  81         # Loop just in case we get a server busy message
  82         while True:
  83             try:
  84                 # Open the URL using our authentication/cookie opener
  85                 return tivo_opener.open(url)
  86
  87             # Do a retry if the TiVo responds that the server is busy
  88             except urllib2.HTTPError, e:
  89                 if e.code == 503:
  90                     time.sleep(5)
  91                     continue
  92
  93                 # Log and throw the error otherwise
  94                 logger.error(e)
  95                 raise
  96
  97     def NPL(self, handler, query):
  98
  99         def getint(thing):
 100             try:
 101                 result = int(thing)
 102             except:
 103                 result = 0
 104             return result
 105
 106         global basic_meta
 107         shows_per_page = 50 # Change this to alter the number of shows returned
 108         folder = ''
 109         FirstAnchor = ''
 110         has_tivodecode = bool(config.get_bin('tivodecode'))
 111
 112         if 'TiVo' in query:
 113             tivoIP = query['TiVo'][0]
 114             tsn = config.tivos_by_ip(tivoIP)
 115
 116             useragent = handler.headers.getheader('User-Agent', '')
 117
 118             attrs = config.tivos[tsn]
 119             tivo_name = attrs.get('name', tivoIP)
 120             tivo_mak = config.get_tsn('tivo_mak', tsn)
 121
 122             protocol = attrs.get('protocol', 'https')
 123             ip_port = '%s:%d' % (tivoIP, attrs.get('port', 443))
 124             path = attrs.get('path', DEFPATH)
 125             if 'Folder' in query:
 126                 folder = query['Folder'][0]
 127                 path += '/' + folder
 128             theurl = '%s://%s%s&ItemCount=%d' % (protocol, ip_port,
 129                                                  path, shows_per_page)
 130             if 'AnchorItem' in query:
 131                 theurl += '&AnchorItem=' + quote(query['AnchorItem'][0])
 132             if 'AnchorOffset' in query:
 133                 theurl += '&AnchorOffset=' + query['AnchorOffset'][0]
 134
 135             if (theurl not in tivo_cache or
 136                 (time.time() - tivo_cache[theurl]['thepage_time']) >= 60):
 137                 # if page is not cached or old then retreive it
 138                 auth_handler.add_password('TiVo DVR', ip_port, 'tivo', tivo_mak)
 139                 try:
 140                     page = self.tivo_open(theurl)
 141                 except IOError, e:
 142                     handler.redir(UNABLE % (tivoIP, e), 10)
 143                     return
 144                 tivo_cache[theurl] = {'thepage': minidom.parse(page),
 145                                       'thepage_time': time.time()}
 146                 page.close()
 147
 148             xmldoc = tivo_cache[theurl]['thepage']
 149             items = xmldoc.getElementsByTagName('Item')
 150             TotalItems = tag_data(xmldoc, 'TiVoContainer/Details/TotalItems')
 151             ItemStart = tag_data(xmldoc, 'TiVoContainer/ItemStart')
 152             ItemCount = tag_data(xmldoc, 'TiVoContainer/ItemCount')
 153             title = tag_data(xmldoc, 'TiVoContainer/Details/Title')
 154             if items:
 155                 FirstAnchor = tag_data(items[0], 'Links/Content/Url')
 156
 157             data = []
 158             for item in items:
 159                 entry = {}
 160                 entry['ContentType'] = tag_data(item, 'Details/ContentType')
 161                 for tag in ('CopyProtected', 'UniqueId'):
 162                     value = tag_data(item, 'Details/' + tag)
 163                     if value:
 164                         entry[tag] = value
 165                 if entry['ContentType'] == 'x-tivo-container/folder':
 166                     entry['Title'] = tag_data(item, 'Details/Title')
 167                     entry['TotalItems'] = tag_data(item, 'Details/TotalItems')
 168                     lc = tag_data(item, 'Details/LastCaptureDate')
 169                     if not lc:
 170                         lc = tag_data(item, 'Details/LastChangeDate')
 171                     entry['LastChangeDate'] = time.strftime('%b %d, %Y',
 172                         time.localtime(int(lc, 16)))
 173                 else:
 174                     keys = {'Icon': 'Links/CustomIcon/Url',
 175                             'Url': 'Links/Content/Url',
 176                             'Details': 'Links/TiVoVideoDetails/Url',
 177                             'SourceSize': 'Details/SourceSize',
 178                             'Duration': 'Details/Duration',
 179                             'CaptureDate': 'Details/CaptureDate'}
 180                     for key in keys:
 181                         value = tag_data(item, keys[key])
 182                         if value:
 183                             entry[key] = value
 184
 185                     rawsize = entry['SourceSize']
 186                     entry['SourceSize'] = metadata.human_size(rawsize)
 187
 188                     if 'Duration' in entry:
 189                         dur = getint(entry['Duration']) / 1000
 190                         entry['Duration'] = ( '%d:%02d:%02d' %
 191                             (dur / 3600, (dur % 3600) / 60, dur % 60) )
 192
 193                     entry['CaptureDate'] = time.strftime('%b %d, %Y',
 194                         time.localtime(int(entry['CaptureDate'], 16)))
 195
 196                     url = entry['Url']
 197                     if url in basic_meta:
 198                         entry.update(basic_meta[url])
 199                     else:
 200                         basic_data = metadata.from_container(item)
 201                         entry.update(basic_data)
 202                         basic_meta[url] = basic_data
 203
 204                 data.append(entry)
 205         else:
 206             data = []
 207             tivoIP = ''
 208             TotalItems = 0
 209             ItemStart = 0
 210             ItemCount = 0
 211             title = ''
 212
 213         if useragent.lower().find('mobile') > 0:
 214             t = Template(CONTAINER_TEMPLATE_MOBILE, filter=EncodeUnicode)
 215         else:
 216             t = Template(CONTAINER_TEMPLATE, filter=EncodeUnicode)
 217         t.escape = escape
 218         t.quote = quote
 219         t.folder = folder
 220         t.status = status
 221         if tivoIP in queue:
 222             t.queue = queue[tivoIP]
 223         t.has_tivodecode = has_tivodecode
 224         t.togo_mpegts = config.is_ts_capable(tsn)
 225         t.tname = tivo_name
 226         t.tivoIP = tivoIP
 227         t.container = handler.cname
 228         t.data = data
 229         t.len = len
 230         t.TotalItems = getint(TotalItems)
 231         t.ItemStart = getint(ItemStart)
 232         t.ItemCount = getint(ItemCount)
 233         t.FirstAnchor = quote(FirstAnchor)
 234         t.shows_per_page = shows_per_page
 235         t.title = title
 236         handler.send_html(str(t), refresh='300')
 237
 238     def get_tivo_file(self, tivoIP, url, mak, togo_path):
 239         # global status
 240         status[url].update({'running': True, 'queued': False})
 241
 242         parse_url = urlparse.urlparse(url)
 243
 244         name = unquote(parse_url[2])[10:].split('.')
 245         try:
 246             id = unquote(parse_url[4]).split('id=')[1]
 247             name.insert(-1, ' - ' + id)
 248         except:
 249             pass
 250         ts = status[url]['ts_format']
 251         if status[url]['decode']:
 252             if ts:
 253                 name[-1] = 'ts'
 254             else:
 255                 name[-1] = 'mpg'
 256         else:
 257             if ts:
 258                 name.insert(-1, ' (TS)')
 259             else:
 260                 name.insert(-1, ' (PS)')
 261         name.insert(-1, '.')
 262         name = ''.join(name)
 263         for ch in BADCHAR:
 264             name = name.replace(ch, BADCHAR[ch])
 265         outfile = os.path.join(togo_path, name)
 266
 267         if status[url]['save']:
 268             meta = basic_meta[url]
 269             details_url = 'https://%s/TiVoVideoDetails?id=%s' % (tivoIP, id)
 270             try:
 271                 handle = self.tivo_open(details_url)
 272                 meta.update(metadata.from_details(handle.read()))
 273                 handle.close()
 274             except:
 275                 pass
 276             metafile = open(outfile + '.txt', 'w')
 277             metadata.dump(metafile, meta)
 278             metafile.close()
 279
 280         auth_handler.add_password('TiVo DVR', url, 'tivo', mak)
 281         try:
 282             if status[url]['ts_format']:
 283                 handle = self.tivo_open(url + '&Format=video/x-tivo-mpeg-ts')
 284             else:
 285                 handle = self.tivo_open(url)
 286         except Exception, msg:
 287             status[url]['running'] = False
 288             status[url]['error'] = str(msg)
 289             return
 290
 291         tivo_name = config.tivos[config.tivos_by_ip(tivoIP)].get('name', tivoIP)
 292
 293         logger.info('[%s] Start getting "%s" from %s' %
 294                     (time.strftime('%d/%b/%Y %H:%M:%S'), outfile, tivo_name))
 295
 296         if status[url]['decode']:
 297             tivodecode_path = config.get_bin('tivodecode')
 298             tcmd = [tivodecode_path, '-m', mak, '-o', outfile, '-']
 299             tivodecode = subprocess.Popen(tcmd, stdin=subprocess.PIPE,
 300                                           bufsize=(512 * 1024))
 301             f = tivodecode.stdin
 302         else:
 303             f = open(outfile, 'wb')
 304         length = 0
 305         start_time = time.time()
 306         last_interval = start_time
 307         now = start_time
 308         try:
 309             while status[url]['running']:
 310                 output = handle.read(1024000)
 311                 if not output:
 312                     break
 313                 length += len(output)
 314                 f.write(output)
 315                 now = time.time()
 316                 elapsed = now - last_interval
 317                 if elapsed >= 5:
 318                     status[url]['rate'] = '%.2f Mb/s' % (length * 8.0 /
 319                         (elapsed * 1024 * 1024))
 320                     status[url]['size'] += length
 321                     length = 0
 322                     last_interval = now
 323             if status[url]['running']:
 324                 status[url]['finished'] = True
 325         except Exception, msg:
 326             status[url]['running'] = False
 327             logger.info(msg)
 328         handle.close()
 329         f.close()
 330         status[url]['size'] += length
 331         if status[url]['running']:
 332             mega_elapsed = (now - start_time) * 1024 * 1024
 333             if mega_elapsed < 1:
 334                 mega_elapsed = 1
 335             size = status[url]['size']
 336             rate = size * 8.0 / mega_elapsed
 337             logger.info('[%s] Done getting "%s" from %s, %d bytes, %.2f Mb/s' %
 338                         (time.strftime('%d/%b/%Y %H:%M:%S'), outfile,
 339                          tivo_name, size, rate))
 340             status[url]['running'] = False
 341         else:
 342             os.remove(outfile)
 343             if status[url]['save']:
 344                 os.remove(outfile + '.txt')
 345             logger.info('[%s] Transfer of "%s" from %s aborted' %
 346                         (time.strftime('%d/%b/%Y %H:%M:%S'), outfile,
 347                          tivo_name))
 348             del status[url]
 349
 350     def process_queue(self, tivoIP, mak, togo_path):
 351         while queue[tivoIP]:
 352             time.sleep(5)
 353             url = queue[tivoIP][0]
 354             self.get_tivo_file(tivoIP, url, mak, togo_path)
 355             queue[tivoIP].pop(0)
 356         del queue[tivoIP]
 357
 358     def ToGo(self, handler, query):
 359         togo_path = config.get_server('togo_path')
 360         for name, data in config.getShares():
 361             if togo_path == name:
 362                 togo_path = data.get('path')
 363         if togo_path:
 364             tivoIP = query['TiVo'][0]
 365             tsn = config.tivos_by_ip(tivoIP)
 366             tivo_mak = config.get_tsn('tivo_mak', tsn)
 367             urls = query.get('Url', [])
 368             decode = 'decode' in query
 369             save = 'save' in query
 370             ts_format = 'ts_format' in query
 371             for theurl in urls:
 372                 status[theurl] = {'running': False, 'error': '', 'rate': '',
 373                                   'queued': True, 'size': 0, 'finished': False,
 374                                   'decode': decode, 'save': save,
 375                                   'ts_format': ts_format}
 376                 if tivoIP in queue:
 377                     queue[tivoIP].append(theurl)
 378                 else:
 379                     queue[tivoIP] = [theurl]
 380                     thread.start_new_thread(ToGo.process_queue,
 381                                             (self, tivoIP, tivo_mak, togo_path))
 382                 logger.info('[%s] Queued "%s" for transfer to %s' %
 383                             (time.strftime('%d/%b/%Y %H:%M:%S'),
 384                              unquote(theurl), togo_path))
 385             urlstring = '<br>'.join([unquote(x) for x in urls])
 386             message = TRANS_QUEUE % (urlstring, togo_path)
 387         else:
 388             message = MISSING
 389         handler.redir(message, 5)
 390
 391     def ToGoStop(self, handler, query):
 392         theurl = query['Url'][0]
 393         status[theurl]['running'] = False
 394         handler.redir(TRANS_STOP % unquote(theurl))
 395
 396     def Unqueue(self, handler, query):
 397         theurl = query['Url'][0]
 398         tivoIP = query['TiVo'][0]
 399         del status[theurl]
 400         queue[tivoIP].remove(theurl)
 401         logger.info('[%s] Removed "%s" from queue' %
 402                     (time.strftime('%d/%b/%Y %H:%M:%S'),
 403                      unquote(theurl)))
 404         handler.redir(UNQUEUE % unquote(theurl))