metadata.py

   1 #!/usr/bin/env python
   2
   3 import os
   4 import subprocess
   5 import sys
   6 from datetime import datetime
   7 from xml.dom import minidom
   8 try:
   9     import plistlib
  10 except:
  11     pass
  12
  13 import mutagen
  14 from lrucache import LRUCache
  15
  16 import config
  17 import plugins.video.transcode
  18
  19 # Something to strip
  20 TRIBUNE_CR = ' Copyright Tribune Media Services, Inc.'
  21
  22 TV_RATINGS = {'TV-Y7': 1, 'TV-Y': 2, 'TV-G': 3, 'TV-PG': 4, 'TV-14': 5,
  23               'TV-MA': 6, 'TV-NR': 7, 'TVY7': 1, 'TVY': 2, 'TVG': 3,
  24               'TVPG': 4, 'TV14': 5, 'TVMA': 6, 'TVNR': 7, 'Y7': 1,
  25               'Y': 2, 'G': 3, 'PG': 4, '14': 5, 'MA': 6, 'NR': 7,
  26               'UNRATED': 7, 'X1': 1, 'X2': 2, 'X3': 3, 'X4': 4, 'X5': 5,
  27               'X6': 6, 'X7': 7}
  28
  29 MPAA_RATINGS = {'G': 1, 'PG': 2, 'PG-13': 3, 'PG13': 3, 'R': 4, 'X': 5,
  30                 'NC-17': 6, 'NC17': 6, 'NR': 8, 'UNRATED': 8, 'G1': 1,
  31                 'P2': 2, 'P3': 3, 'R4': 4, 'X5': 5, 'N6': 6, 'N8': 8}
  32
  33 STAR_RATINGS = {'1': 1, '1.5': 2, '2': 3, '2.5': 4, '3': 5, '3.5': 6,
  34                 '4': 7, '*': 1, '**': 3, '***': 5, '****': 7}
  35
  36 HUMAN = {'mpaaRating': {1: 'G', 2: 'PG', 3: 'PG-13', 4: 'R', 5: 'X',
  37                         6: 'NC-17', 8: 'NR'},
  38          'tvRating': {1: 'Y7', 2: 'Y', 3: 'G', 4: 'PG', 5: '14',
  39                       6: 'MA', 7: 'NR'},
  40          'starRating': {1: '1', 2: '1.5', 3: '2', 4: '2.5', 5: '3',
  41                         6: '3.5', 7: '4'}}
  42
  43 BOM = '\xef\xbb\xbf'
  44
  45 tivo_cache = LRUCache(50)
  46 mp4_cache = LRUCache(50)
  47 dvrms_cache = LRUCache(50)
  48
  49 mswindows = (sys.platform == "win32")
  50
  51 def get_mpaa(rating):
  52     return HUMAN['mpaaRating'].get(rating, 'NR')
  53
  54 def get_tv(rating):
  55     return HUMAN['tvRating'].get(rating, 'NR')
  56
  57 def get_stars(rating):
  58     return HUMAN['starRating'].get(rating, '')
  59
  60 def tag_data(element, tag):
  61     for name in tag.split('/'):
  62         new_element = element.getElementsByTagName(name)
  63         if not new_element:
  64             return ''
  65         element = new_element[0]
  66     if not element.firstChild:
  67         return ''
  68     return element.firstChild.data
  69
  70 def _vtag_data(element, tag):
  71     for name in tag.split('/'):
  72         new_element = element.getElementsByTagName(name)
  73         if not new_element:
  74             return []
  75         element = new_element[0]
  76     elements = element.getElementsByTagName('element')
  77     return [x.firstChild.data for x in elements if x.firstChild]
  78
  79 def _tag_value(element, tag):
  80     item = element.getElementsByTagName(tag)
  81     if item:
  82         value = item[0].attributes['value'].value
  83         return int(value[0])
  84
  85 def from_moov(full_path):
  86     if full_path in mp4_cache:
  87         return mp4_cache[full_path]
  88
  89     metadata = {}
  90     len_desc = 0
  91
  92     try:
  93         mp4meta = mutagen.File(unicode(full_path, 'utf-8'))
  94         assert(mp4meta)
  95     except:
  96         mp4_cache[full_path] = {}
  97         return {}
  98
  99     # The following 1-to-1 correspondence of atoms to pyTivo
 100     # variables is TV-biased
 101     keys = {'tvnn': 'callsign', 'tven': 'episodeNumber',
 102             'tvsh': 'seriesTitle'}
 103
 104     for key, value in mp4meta.items():
 105         if type(value) == list:
 106             value = value[0]
 107         if key == 'stik':
 108             metadata['isEpisode'] = ['false', 'true'][value == 'TV Show']
 109         elif key in keys:
 110             metadata[keys[key]] = value
 111         # These keys begin with the copyright symbol \xA9
 112         elif key == '\xa9day':
 113             if len(value) == 4:
 114                 value += '-01-01T16:00:00Z'
 115             metadata['originalAirDate'] = value
 116             #metadata['time'] = value
 117         elif key in ['\xa9gen', 'gnre']:
 118             for k in ('vProgramGenre', 'vSeriesGenre'):
 119                 if k in metadata:
 120                     metadata[k].append(value)
 121                 else:
 122                     metadata[k] = [value]
 123         elif key == '\xa9nam':
 124             if 'tvsh' in mp4meta:
 125                 metadata['episodeTitle'] = value
 126             else:
 127                 metadata['title'] = value
 128
 129         # Description in desc, cmt, and/or ldes tags. Keep the longest.
 130         elif key in ['desc', '\xa9cmt', 'ldes'] and len(value) > len_desc:
 131             metadata['description'] = value
 132             len_desc = len(value)
 133
 134         # A common custom "reverse DNS format" tag
 135         elif (key == '----:com.apple.iTunes:iTunEXTC' and
 136               ('us-tv' in value or 'mpaa' in value)):
 137             rating = value.split("|")[1].upper()
 138             if rating in TV_RATINGS and 'us-tv' in value:
 139                 metadata['tvRating'] = TV_RATINGS[rating]
 140             elif rating in MPAA_RATINGS and 'mpaa' in value:
 141                 metadata['mpaaRating'] = MPAA_RATINGS[rating]
 142
 143         # Actors, directors, producers, AND screenwriters may be in a long
 144         # embedded XML plist.
 145         elif (key == '----:com.apple.iTunes:iTunMOVI' and
 146               'plistlib' in sys.modules):
 147             items = {'cast': 'vActor', 'directors': 'vDirector',
 148                      'producers': 'vProducer', 'screenwriters': 'vWriter'}
 149             data = plistlib.readPlistFromString(value)
 150             for item in items:
 151                 if item in data:
 152                     metadata[items[item]] = [x['name'] for x in data[item]]
 153
 154     mp4_cache[full_path] = metadata
 155     return metadata
 156
 157 def from_mscore(rawmeta):
 158     metadata = {}
 159     keys = {'title': ['Title'],
 160             'description': ['Description', 'WM/SubTitleDescription'],
 161             'episodeTitle': ['WM/SubTitle'],
 162             'callsign': ['WM/MediaStationCallSign'],
 163             'displayMajorNumber': ['WM/MediaOriginalChannel'],
 164             'originalAirDate': ['WM/MediaOriginalBroadcastDateTime'],
 165             'rating': ['WM/ParentalRating'],
 166             'credits': ['WM/MediaCredits'], 'genre': ['WM/Genre']}
 167
 168     for tagname in keys:
 169         for tag in keys[tagname]:
 170             try:
 171                 if tag in rawmeta:
 172                     value = str(rawmeta[tag][0])
 173                     if value:
 174                         metadata[tagname] = value
 175             except:
 176                 pass
 177
 178     if 'episodeTitle' in metadata and 'title' in metadata:
 179         metadata['seriesTitle'] = metadata['title']
 180     if 'genre' in metadata:
 181         value = metadata['genre'].split(',')
 182         metadata['vProgramGenre'] = value
 183         metadata['vSeriesGenre'] = value
 184         del metadata['genre']
 185     if 'credits' in metadata:
 186         value = [x.split('/') for x in metadata['credits'].split(';')]
 187         if len(value) > 3:
 188             metadata['vActor'] = [x for x in (value[0] + value[3]) if x]
 189             metadata['vDirector'] = [x for x in value[1] if x]
 190         del metadata['credits']
 191     if 'rating' in metadata:
 192         rating = metadata['rating']
 193         if rating in TV_RATINGS:
 194             metadata['tvRating'] = TV_RATINGS[rating]
 195         del metadata['rating']
 196
 197     return metadata
 198
 199 def from_dvrms(full_path):
 200     if full_path in dvrms_cache:
 201         return dvrms_cache[full_path]
 202
 203     try:
 204         rawmeta = mutagen.File(unicode(full_path, 'utf-8'))
 205         assert(rawmeta)
 206     except:
 207         dvrms_cache[full_path] = {}
 208         return {}
 209
 210     metadata = from_mscore(rawmeta)
 211     dvrms_cache[full_path] = metadata
 212     return metadata
 213
 214 def from_eyetv(full_path):
 215     keys = {'TITLE': 'title', 'SUBTITLE': 'episodeTitle',
 216             'DESCRIPTION': 'description', 'YEAR': 'movieYear',
 217             'EPISODENUM': 'episodeNumber'}
 218     metadata = {}
 219     path, name = os.path.split(unicode(full_path, 'utf-8'))
 220     eyetvp = [x for x in os.listdir(path) if x.endswith('.eyetvp')][0]
 221     eyetvp = os.path.join(path, eyetvp)
 222     eyetv = plistlib.readPlist(eyetvp)
 223     if 'epg info' in eyetv:
 224         info = eyetv['epg info']
 225         for key in keys:
 226             if info[key]:
 227                 metadata[keys[key]] = info[key]
 228         if info['SUBTITLE']:
 229             metadata['seriesTitle'] = info['TITLE']
 230         if info['ACTORS']:
 231             metadata['vActor'] = [x.strip() for x in info['ACTORS'].split(',')]
 232         if info['DIRECTOR']:
 233             metadata['vDirector'] = [info['DIRECTOR']]
 234
 235         for ptag, etag, ratings in [('tvRating', 'TV_RATING', TV_RATINGS),
 236                               ('mpaaRating', 'MPAA_RATING', MPAA_RATINGS),
 237                               ('starRating', 'STAR_RATING', STAR_RATINGS)]:
 238            x = info[etag].upper()
 239            if x and x in ratings:
 240                metadata[ptag] = ratings[x]
 241
 242         # movieYear must be set for the mpaa/star ratings to work
 243         if (('mpaaRating' in metadata or 'starRating' in metadata) and
 244             'movieYear' not in metadata):
 245             metadata['movieYear'] = eyetv['info']['start'].year
 246     return metadata
 247
 248 def from_text(full_path):
 249     metadata = {}
 250     full_path = unicode(full_path, 'utf-8')
 251     path, name = os.path.split(full_path)
 252     title, ext = os.path.splitext(name)
 253
 254     for metafile in [os.path.join(path, title) + '.properties',
 255                      os.path.join(path, 'default.txt'), full_path + '.txt',
 256                      os.path.join(path, '.meta', 'default.txt'),
 257                      os.path.join(path, '.meta', name) + '.txt']:
 258         if os.path.exists(metafile):
 259             sep = ':='[metafile.endswith('.properties')]
 260             for line in file(metafile, 'U'):
 261                 if line.startswith(BOM):
 262                     line = line[3:]
 263                 if line.strip().startswith('#') or not sep in line:
 264                     continue
 265                 key, value = [x.strip() for x in line.split(sep, 1)]
 266                 if not key or not value:
 267                     continue
 268                 if key.startswith('v'):
 269                     if key in metadata:
 270                         metadata[key].append(value)
 271                     else:
 272                         metadata[key] = [value]
 273                 else:
 274                     metadata[key] = value
 275
 276     for rating, ratings in [('tvRating', TV_RATINGS),
 277                             ('mpaaRating', MPAA_RATINGS),
 278                             ('starRating', STAR_RATINGS)]:
 279         x = metadata.get(rating, '').upper()
 280         if x in ratings:
 281             metadata[rating] = ratings[x]
 282
 283     return metadata
 284
 285 def basic(full_path):
 286     base_path, name = os.path.split(full_path)
 287     title, ext = os.path.splitext(name)
 288     mtime = os.stat(unicode(full_path, 'utf-8')).st_mtime
 289     if (mtime < 0):
 290         mtime = 0
 291     originalAirDate = datetime.fromtimestamp(mtime)
 292
 293     metadata = {'title': title,
 294                 'originalAirDate': originalAirDate.isoformat()}
 295     ext = ext.lower()
 296     if ext in ['.mp4', '.m4v', '.mov']:
 297         metadata.update(from_moov(full_path))
 298     elif ext in ['.dvr-ms', '.asf', '.wmv']:
 299         metadata.update(from_dvrms(full_path))
 300     elif 'plistlib' in sys.modules and base_path.endswith('.eyetv'):
 301         metadata.update(from_eyetv(full_path))
 302     metadata.update(from_text(full_path))
 303
 304     return metadata
 305
 306 def from_container(xmldoc):
 307     metadata = {}
 308
 309     keys = {'title': 'Title', 'episodeTitle': 'EpisodeTitle',
 310             'description': 'Description', 'seriesId': 'SeriesId',
 311             'episodeNumber': 'EpisodeNumber', 'tvRating': 'TvRating',
 312             'displayMajorNumber': 'SourceChannel', 'callsign': 'SourceStation',
 313             'showingBits': 'ShowingBits', 'mpaaRating': 'MpaaRating'}
 314
 315     details = xmldoc.getElementsByTagName('Details')[0]
 316
 317     for key in keys:
 318         data = tag_data(details, keys[key])
 319         if data:
 320             if key == 'description':
 321                 data = data.replace(TRIBUNE_CR, '')
 322             elif key == 'tvRating':
 323                 data = int(data)
 324             elif key == 'displayMajorNumber':
 325                 if '-' in data:
 326                     data, metadata['displayMinorNumber'] = data.split('-')
 327             metadata[key] = data
 328
 329     return metadata
 330
 331 def from_details(xml):
 332     metadata = {}
 333
 334     xmldoc = minidom.parse(xml)
 335     showing = xmldoc.getElementsByTagName('showing')[0]
 336     program = showing.getElementsByTagName('program')[0]
 337
 338     items = {'description': 'program/description',
 339              'title': 'program/title',
 340              'episodeTitle': 'program/episodeTitle',
 341              'episodeNumber': 'program/episodeNumber',
 342              'seriesId': 'program/series/uniqueId',
 343              'seriesTitle': 'program/series/seriesTitle',
 344              'originalAirDate': 'program/originalAirDate',
 345              'isEpisode': 'program/isEpisode',
 346              'movieYear': 'program/movieYear',
 347              'partCount': 'partCount',
 348              'partIndex': 'partIndex',
 349              'time': 'time'}
 350
 351     for item in items:
 352         data = tag_data(showing, items[item])
 353         if data:
 354             if item == 'description':
 355                 data = data.replace(TRIBUNE_CR, '')
 356             metadata[item] = data
 357
 358     vItems = ['vActor', 'vChoreographer', 'vDirector',
 359               'vExecProducer', 'vProgramGenre', 'vGuestStar',
 360               'vHost', 'vProducer', 'vWriter']
 361
 362     for item in vItems:
 363         data = _vtag_data(program, item)
 364         if data:
 365             metadata[item] = data
 366
 367     sb = showing.getElementsByTagName('showingBits')
 368     if sb:
 369         metadata['showingBits'] = sb[0].attributes['value'].value
 370
 371     #for tag in ['starRating', 'mpaaRating', 'colorCode']:
 372     for tag in ['starRating', 'mpaaRating']:
 373         value = _tag_value(program, tag)
 374         if value:
 375             metadata[tag] = value
 376
 377     rating = _tag_value(showing, 'tvRating')
 378     if rating:
 379         metadata['tvRating'] = rating
 380
 381     return metadata
 382
 383 def from_tivo(full_path):
 384     if full_path in tivo_cache:
 385         return tivo_cache[full_path]
 386
 387     tdcat_path = config.get_bin('tdcat')
 388     tivo_mak = config.get_server('tivo_mak')
 389     try:
 390         assert(tdcat_path and tivo_mak)
 391         fname = unicode(full_path, 'utf-8')
 392         if mswindows:
 393             fname = fname.encode('iso8859-1')
 394         tcmd = [tdcat_path, '-m', tivo_mak, '-2', fname]
 395         tdcat = subprocess.Popen(tcmd, stdout=subprocess.PIPE)
 396         metadata = from_details(tdcat.stdout)
 397         tivo_cache[full_path] = metadata
 398     except:
 399         metadata = {}
 400
 401     return metadata
 402
 403 def force_utf8(text):
 404     if type(text) == str:
 405         try:
 406             text = text.decode('utf8')
 407         except:
 408             if sys.platform == 'darwin':
 409                 text = text.decode('macroman')
 410             else:
 411                 text = text.decode('iso8859-1')
 412     return text.encode('utf-8')
 413
 414 def dump(output, metadata):
 415     for key in metadata:
 416         value = metadata[key]
 417         if type(value) == list:
 418             for item in value:
 419                 output.write('%s: %s\n' % (key, item.encode('utf-8')))
 420         else:
 421             if key in HUMAN and value in HUMAN[key]:
 422                 output.write('%s: %s\n' % (key, HUMAN[key][value]))
 423             else:
 424                 output.write('%s: %s\n' % (key, value.encode('utf-8')))
 425
 426 if __name__ == '__main__':
 427     if len(sys.argv) > 1:
 428         metadata = {}
 429         fname = force_utf8(sys.argv[1])
 430         ext = os.path.splitext(fname)[1].lower()
 431         if ext == '.tivo':
 432             config.init([])
 433             metadata.update(from_tivo(fname))
 434         elif ext in ['.mp4', '.m4v', '.mov']:
 435             metadata.update(from_moov(fname))
 436         elif ext in ['.dvr-ms', '.asf', '.wmv']:
 437             metadata.update(from_dvrms(fname))
 438         elif ext == '.wtv':
 439             vInfo = plugins.video.transcode.video_info(fname)
 440             metadata.update(from_mscore(vInfo['rawmeta']))
 441         dump(sys.stdout, metadata)