plugins/togo/togo.py

   1 import cookielib
   2 import logging
   3 import os
   4 import subprocess
   5 import thread
   6 import time
   7 import urllib2
   8 import urlparse
   9 from urllib import quote, unquote
  10 from xml.dom import minidom
  11 from xml.sax.saxutils import escape
  12
  13 from Cheetah.Template import Template
  14
  15 import config
  16 import metadata
  17 from plugin import EncodeUnicode, Plugin
  18
  19 logger = logging.getLogger('pyTivo.togo')
  20 tag_data = metadata.tag_data
  21
  22 SCRIPTDIR = os.path.dirname(__file__)
  23
  24 CLASS_NAME = 'ToGo'
  25
  26 # Characters to remove from filenames
  27
  28 BADCHAR = {'\\': '-', '/': '-', ':': ' -', ';': ',', '*': '.',
  29            '?': '.', '!': '.', '"': "'", '<': '(', '>': ')', '|': ' '}
  30
  31 # Some error/status message templates
  32
  33 MISSING = """<h3>Missing Data</h3> <p>You must set both "tivo_mak" and
  34 "togo_path" before using this function.</p>"""
  35
  36 TRANS_QUEUE = """<h3>Queued for Transfer</h3> <p>%s</p> <p>queued for
  37 transfer to:</p> <p>%s</p>"""
  38
  39 TRANS_STOP = """<h3>Transfer Stopped</h3> <p>Your transfer of:</p>
  40 <p>%s</p> <p>has been stopped.</p>"""
  41
  42 UNQUEUE = """<h3>Removed from Queue</h3> <p>%s</p> <p>has been removed
  43 from the queue.</p>"""
  44
  45 UNABLE = """<h3>Unable to Connect to TiVo</h3> <p>pyTivo was unable to
  46 connect to the TiVo at %s.</p> <p>This is most likely caused by an
  47 incorrect Media Access Key. Please return to the Settings page and
  48 double check your <b>tivo_mak</b> setting.</p> <pre>%s</pre>"""
  49
  50 # Preload the templates
  51 def tmpl(name):
  52     return file(os.path.join(SCRIPTDIR, 'templates', name), 'rb').read()
  53
  54 CONTAINER_TEMPLATE_MOBILE = tmpl('npl_mob.tmpl')
  55 CONTAINER_TEMPLATE = tmpl('npl.tmpl')
  56
  57 status = {} # Global variable to control download threads
  58 tivo_cache = {} # Cache of TiVo NPL
  59 queue = {} # Recordings to download -- list per TiVo
  60 basic_meta = {} # Data from NPL, parsed, indexed by progam URL
  61
  62 def null_cookie(name, value):
  63     return cookielib.Cookie(0, name, value, None, False, '', False,
  64         False, '', False, False, None, False, None, None, None)
  65
  66 auth_handler = urllib2.HTTPPasswordMgrWithDefaultRealm()
  67 cj = cookielib.CookieJar()
  68 cj.set_cookie(null_cookie('sid', 'ADEADDA7EDEBAC1E'))
  69 tivo_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj),
  70                                    urllib2.HTTPBasicAuthHandler(auth_handler),
  71                                    urllib2.HTTPDigestAuthHandler(auth_handler))
  72
  73 class ToGo(Plugin):
  74     CONTENT_TYPE = 'text/html'
  75
  76     def tivo_open(self, url):
  77         # Loop just in case we get a server busy message
  78         while True:
  79             try:
  80                 # Open the URL using our authentication/cookie opener
  81                 return tivo_opener.open(url)
  82
  83             # Do a retry if the TiVo responds that the server is busy
  84             except urllib2.HTTPError, e:
  85                 if e.code == 503:
  86                     time.sleep(5)
  87                     continue
  88
  89                 # Log and throw the error otherwise
  90                 logger.error(e)
  91                 raise
  92
  93     def NPL(self, handler, query):
  94
  95         def getint(thing):
  96             try:
  97                 result = int(thing)
  98             except:
  99                 result = 0
 100             return result
 101
 102         global basic_meta
 103         shows_per_page = 50 # Change this to alter the number of shows returned
 104         folder = ''
 105         FirstAnchor = ''
 106         has_tivodecode = bool(config.get_bin('tivodecode'))
 107
 108         if 'TiVo' in query:
 109             tivoIP = query['TiVo'][0]
 110             tsn = config.tivos_by_ip(tivoIP)
 111
 112             useragent = handler.headers.getheader('User-Agent', '')
 113
 114             tivo_name = config.tivos[tsn].get('name', tivoIP)
 115             tivo_mak = config.get_tsn('tivo_mak', tsn)
 116             if 'port' in config.tivos[tsn]:
 117                 ip_port = '%s:%d' % (tivoIP, config.tivos[tsn]['port'])
 118             else:
 119                 ip_port = tivoIP
 120             theurl = ('https://' + ip_port +
 121                       '/TiVoConnect?Command=QueryContainer&ItemCount=' +
 122                       str(shows_per_page) + '&Container=/NowPlaying')
 123             if 'Folder' in query:
 124                 folder += query['Folder'][0]
 125                 theurl += '/' + folder
 126             if 'AnchorItem' in query:
 127                 theurl += '&AnchorItem=' + quote(query['AnchorItem'][0])
 128             if 'AnchorOffset' in query:
 129                 theurl += '&AnchorOffset=' + query['AnchorOffset'][0]
 130
 131             if (theurl not in tivo_cache or
 132                 (time.time() - tivo_cache[theurl]['thepage_time']) >= 60):
 133                 # if page is not cached or old then retreive it
 134                 auth_handler.add_password('TiVo DVR', ip_port, 'tivo', tivo_mak)
 135                 try:
 136                     page = self.tivo_open(theurl)
 137                 except IOError, e:
 138                     handler.redir(UNABLE % (tivoIP, e), 10)
 139                     return
 140                 tivo_cache[theurl] = {'thepage': minidom.parse(page),
 141                                       'thepage_time': time.time()}
 142                 page.close()
 143
 144             xmldoc = tivo_cache[theurl]['thepage']
 145             items = xmldoc.getElementsByTagName('Item')
 146             TotalItems = tag_data(xmldoc, 'TiVoContainer/Details/TotalItems')
 147             ItemStart = tag_data(xmldoc, 'TiVoContainer/ItemStart')
 148             ItemCount = tag_data(xmldoc, 'TiVoContainer/ItemCount')
 149             title = tag_data(xmldoc, 'TiVoContainer/Details/Title')
 150             if items:
 151                 FirstAnchor = tag_data(items[0], 'Links/Content/Url')
 152
 153             data = []
 154             for item in items:
 155                 entry = {}
 156                 entry['ContentType'] = tag_data(item, 'Details/ContentType')
 157                 for tag in ('CopyProtected', 'UniqueId'):
 158                     value = tag_data(item, 'Details/' + tag)
 159                     if value:
 160                         entry[tag] = value
 161                 if entry['ContentType'] == 'x-tivo-container/folder':
 162                     entry['Title'] = tag_data(item, 'Details/Title')
 163                     entry['TotalItems'] = tag_data(item, 'Details/TotalItems')
 164                     lc = tag_data(item, 'Details/LastCaptureDate')
 165                     if not lc:
 166                         lc = tag_data(item, 'Details/LastChangeDate')
 167                     entry['LastChangeDate'] = time.strftime('%b %d, %Y',
 168                         time.localtime(int(lc, 16)))
 169                 else:
 170                     keys = {'Icon': 'Links/CustomIcon/Url',
 171                             'Url': 'Links/Content/Url',
 172                             'Details': 'Links/TiVoVideoDetails/Url',
 173                             'SourceSize': 'Details/SourceSize',
 174                             'Duration': 'Details/Duration',
 175                             'CaptureDate': 'Details/CaptureDate'}
 176                     for key in keys:
 177                         value = tag_data(item, keys[key])
 178                         if value:
 179                             entry[key] = value
 180
 181                     rawsize = entry['SourceSize']
 182                     entry['SourceSize'] = metadata.human_size(rawsize)
 183
 184                     if 'Duration' in entry:
 185                         dur = getint(entry['Duration']) / 1000
 186                         entry['Duration'] = ( '%d:%02d:%02d' %
 187                             (dur / 3600, (dur % 3600) / 60, dur % 60) )
 188
 189                     entry['CaptureDate'] = time.strftime('%b %d, %Y',
 190                         time.localtime(int(entry['CaptureDate'], 16)))
 191
 192                     url = entry['Url']
 193                     if url in basic_meta:
 194                         entry.update(basic_meta[url])
 195                     else:
 196                         basic_data = metadata.from_container(item)
 197                         entry.update(basic_data)
 198                         basic_meta[url] = basic_data
 199
 200                 data.append(entry)
 201         else:
 202             data = []
 203             tivoIP = ''
 204             TotalItems = 0
 205             ItemStart = 0
 206             ItemCount = 0
 207             title = ''
 208
 209         if useragent.lower().find('mobile') > 0:
 210             t = Template(CONTAINER_TEMPLATE_MOBILE, filter=EncodeUnicode)
 211         else:
 212             t = Template(CONTAINER_TEMPLATE, filter=EncodeUnicode)
 213         t.escape = escape
 214         t.quote = quote
 215         t.folder = folder
 216         t.status = status
 217         if tivoIP in queue:
 218             t.queue = queue[tivoIP]
 219         t.has_tivodecode = has_tivodecode
 220         t.togo_mpegts = config.is_ts_capable(tsn)
 221         t.tname = tivo_name
 222         t.tivoIP = tivoIP
 223         t.container = handler.cname
 224         t.data = data
 225         t.len = len
 226         t.TotalItems = getint(TotalItems)
 227         t.ItemStart = getint(ItemStart)
 228         t.ItemCount = getint(ItemCount)
 229         t.FirstAnchor = quote(FirstAnchor)
 230         t.shows_per_page = shows_per_page
 231         t.title = title
 232         handler.send_html(str(t), refresh='300')
 233
 234     def get_tivo_file(self, tivoIP, url, mak, togo_path):
 235         # global status
 236         status[url].update({'running': True, 'queued': False})
 237
 238         parse_url = urlparse.urlparse(url)
 239
 240         name = unquote(parse_url[2])[10:].split('.')
 241         try:
 242             id = unquote(parse_url[4]).split('id=')[1]
 243             name.insert(-1, ' - ' + id)
 244         except:
 245             pass
 246         ts = status[url]['ts_format']
 247         if status[url]['decode']:
 248             if ts:
 249                 name[-1] = 'ts'
 250             else:
 251                 name[-1] = 'mpg'
 252         else:
 253             if ts:
 254                 name.insert(-1, ' (TS)')
 255             else:
 256                 name.insert(-1, ' (PS)')
 257         name.insert(-1, '.')
 258         name = ''.join(name)
 259         for ch in BADCHAR:
 260             name = name.replace(ch, BADCHAR[ch])
 261         outfile = os.path.join(togo_path, name)
 262
 263         if status[url]['save']:
 264             meta = basic_meta[url]
 265             details_url = 'https://%s/TiVoVideoDetails?id=%s' % (tivoIP, id)
 266             try:
 267                 handle = self.tivo_open(details_url)
 268                 meta.update(metadata.from_details(handle.read()))
 269                 handle.close()
 270             except:
 271                 pass
 272             metafile = open(outfile + '.txt', 'w')
 273             metadata.dump(metafile, meta)
 274             metafile.close()
 275
 276         auth_handler.add_password('TiVo DVR', url, 'tivo', mak)
 277         try:
 278             if status[url]['ts_format']:
 279                 handle = self.tivo_open(url + '&Format=video/x-tivo-mpeg-ts')
 280             else:
 281                 handle = self.tivo_open(url)
 282         except Exception, msg:
 283             status[url]['running'] = False
 284             status[url]['error'] = str(msg)
 285             return
 286
 287         tivo_name = config.tivos[config.tivos_by_ip(tivoIP)].get('name', tivoIP)
 288
 289         logger.info('[%s] Start getting "%s" from %s' %
 290                     (time.strftime('%d/%b/%Y %H:%M:%S'), outfile, tivo_name))
 291
 292         if status[url]['decode']:
 293             tivodecode_path = config.get_bin('tivodecode')
 294             tcmd = [tivodecode_path, '-m', mak, '-o', outfile, '-']
 295             tivodecode = subprocess.Popen(tcmd, stdin=subprocess.PIPE,
 296                                           bufsize=(512 * 1024))
 297             f = tivodecode.stdin
 298         else:
 299             f = open(outfile, 'wb')
 300         length = 0
 301         start_time = time.time()
 302         last_interval = start_time
 303         now = start_time
 304         try:
 305             while status[url]['running']:
 306                 output = handle.read(1024000)
 307                 if not output:
 308                     break
 309                 length += len(output)
 310                 f.write(output)
 311                 now = time.time()
 312                 elapsed = now - last_interval
 313                 if elapsed >= 5:
 314                     status[url]['rate'] = '%.2f Mb/s' % (length * 8.0 /
 315                         (elapsed * 1024 * 1024))
 316                     status[url]['size'] += length
 317                     length = 0
 318                     last_interval = now
 319             if status[url]['running']:
 320                 status[url]['finished'] = True
 321         except Exception, msg:
 322             status[url]['running'] = False
 323             logger.info(msg)
 324         handle.close()
 325         f.close()
 326         status[url]['size'] += length
 327         if status[url]['running']:
 328             mega_elapsed = (now - start_time) * 1024 * 1024
 329             if mega_elapsed < 1:
 330                 mega_elapsed = 1
 331             size = status[url]['size']
 332             rate = size * 8.0 / mega_elapsed
 333             logger.info('[%s] Done getting "%s" from %s, %d bytes, %.2f Mb/s' %
 334                         (time.strftime('%d/%b/%Y %H:%M:%S'), outfile,
 335                          tivo_name, size, rate))
 336             status[url]['running'] = False
 337         else:
 338             os.remove(outfile)
 339             if status[url]['save']:
 340                 os.remove(outfile + '.txt')
 341             logger.info('[%s] Transfer of "%s" from %s aborted' %
 342                         (time.strftime('%d/%b/%Y %H:%M:%S'), outfile,
 343                          tivo_name))
 344             del status[url]
 345
 346     def process_queue(self, tivoIP, mak, togo_path):
 347         while queue[tivoIP]:
 348             time.sleep(5)
 349             url = queue[tivoIP][0]
 350             self.get_tivo_file(tivoIP, url, mak, togo_path)
 351             queue[tivoIP].pop(0)
 352         del queue[tivoIP]
 353
 354     def ToGo(self, handler, query):
 355         togo_path = config.get_server('togo_path')
 356         for name, data in config.getShares():
 357             if togo_path == name:
 358                 togo_path = data.get('path')
 359         if togo_path:
 360             tivoIP = query['TiVo'][0]
 361             tsn = config.tivos_by_ip(tivoIP)
 362             tivo_mak = config.get_tsn('tivo_mak', tsn)
 363             urls = query.get('Url', [])
 364             decode = 'decode' in query
 365             save = 'save' in query
 366             ts_format = 'ts_format' in query
 367             for theurl in urls:
 368                 status[theurl] = {'running': False, 'error': '', 'rate': '',
 369                                   'queued': True, 'size': 0, 'finished': False,
 370                                   'decode': decode, 'save': save,
 371                                   'ts_format': ts_format}
 372                 if tivoIP in queue:
 373                     queue[tivoIP].append(theurl)
 374                 else:
 375                     queue[tivoIP] = [theurl]
 376                     thread.start_new_thread(ToGo.process_queue,
 377                                             (self, tivoIP, tivo_mak, togo_path))
 378                 logger.info('[%s] Queued "%s" for transfer to %s' %
 379                             (time.strftime('%d/%b/%Y %H:%M:%S'),
 380                              unquote(theurl), togo_path))
 381             urlstring = '<br>'.join([unquote(x) for x in urls])
 382             message = TRANS_QUEUE % (urlstring, togo_path)
 383         else:
 384             message = MISSING
 385         handler.redir(message, 5)
 386
 387     def ToGoStop(self, handler, query):
 388         theurl = query['Url'][0]
 389         status[theurl]['running'] = False
 390         handler.redir(TRANS_STOP % unquote(theurl))
 391
 392     def Unqueue(self, handler, query):
 393         theurl = query['Url'][0]
 394         tivoIP = query['TiVo'][0]
 395         del status[theurl]
 396         queue[tivoIP].remove(theurl)
 397         logger.info('[%s] Removed "%s" from queue' %
 398                     (time.strftime('%d/%b/%Y %H:%M:%S'),
 399                      unquote(theurl)))
 400         handler.redir(UNQUEUE % unquote(theurl))