metadata.py

   1 #!/usr/bin/env python
   2
   3 import os
   4 import subprocess
   5 import sys
   6 from datetime import datetime
   7 from xml.dom import minidom
   8 try:
   9     import plistlib
  10 except:
  11     pass
  12
  13 import mutagen
  14 from lrucache import LRUCache
  15
  16 import config
  17 import plugins.video.transcode
  18
  19 # Something to strip
  20 TRIBUNE_CR = ' Copyright Tribune Media Services, Inc.'
  21
  22 TV_RATINGS = {'TV-Y7': 1, 'TV-Y': 2, 'TV-G': 3, 'TV-PG': 4, 'TV-14': 5,
  23               'TV-MA': 6, 'TV-NR': 7, 'TVY7': 1, 'TVY': 2, 'TVG': 3,
  24               'TVPG': 4, 'TV14': 5, 'TVMA': 6, 'TVNR': 7, 'Y7': 1,
  25               'Y': 2, 'G': 3, 'PG': 4, '14': 5, 'MA': 6, 'NR': 7,
  26               'UNRATED': 7, 'X1': 1, 'X2': 2, 'X3': 3, 'X4': 4, 'X5': 5,
  27               'X6': 6, 'X7': 7}
  28
  29 MPAA_RATINGS = {'G': 1, 'PG': 2, 'PG-13': 3, 'PG13': 3, 'R': 4, 'X': 5,
  30                 'NC-17': 6, 'NC17': 6, 'NR': 8, 'UNRATED': 8, 'G1': 1,
  31                 'P2': 2, 'P3': 3, 'R4': 4, 'X5': 5, 'N6': 6, 'N8': 8}
  32
  33 STAR_RATINGS = {'1': 1, '1.5': 2, '2': 3, '2.5': 4, '3': 5, '3.5': 6,
  34                 '4': 7, '*': 1, '**': 3, '***': 5, '****': 7}
  35
  36 HUMAN = {'mpaaRating': {1: 'G', 2: 'PG', 3: 'PG-13', 4: 'R', 5: 'X',
  37                         6: 'NC-17', 8: 'NR'},
  38          'tvRating': {1: 'Y7', 2: 'Y', 3: 'G', 4: 'PG', 5: '14',
  39                       6: 'MA', 7: 'NR'},
  40          'starRating': {1: '1', 2: '1.5', 3: '2', 4: '2.5', 5: '3',
  41                         6: '3.5', 7: '4'}}
  42
  43 BOM = '\xef\xbb\xbf'
  44
  45 tivo_cache = LRUCache(50)
  46 mp4_cache = LRUCache(50)
  47 dvrms_cache = LRUCache(50)
  48
  49 mswindows = (sys.platform == "win32")
  50
  51 def get_mpaa(rating):
  52     return HUMAN['mpaaRating'].get(rating, 'NR')
  53
  54 def get_tv(rating):
  55     return HUMAN['tvRating'].get(rating, 'NR')
  56
  57 def get_stars(rating):
  58     return HUMAN['starRating'].get(rating, '')
  59
  60 def tag_data(element, tag):
  61     for name in tag.split('/'):
  62         new_element = element.getElementsByTagName(name)
  63         if not new_element:
  64             return ''
  65         element = new_element[0]
  66     if not element.firstChild:
  67         return ''
  68     return element.firstChild.data
  69
  70 def _vtag_data(element, tag):
  71     for name in tag.split('/'):
  72         new_element = element.getElementsByTagName(name)
  73         if not new_element:
  74             return []
  75         element = new_element[0]
  76     elements = element.getElementsByTagName('element')
  77     return [x.firstChild.data for x in elements if x.firstChild]
  78
  79 def _tag_value(element, tag):
  80     item = element.getElementsByTagName(tag)
  81     if item:
  82         value = item[0].attributes['value'].value
  83         return int(value[0])
  84
  85 def from_moov(full_path):
  86     if full_path in mp4_cache:
  87         return mp4_cache[full_path]
  88
  89     metadata = {}
  90     len_desc = 0
  91
  92     try:
  93         mp4meta = mutagen.File(unicode(full_path, 'utf-8'))
  94         assert(mp4meta)
  95     except:
  96         mp4_cache[full_path] = {}
  97         return {}
  98
  99     # The following 1-to-1 correspondence of atoms to pyTivo
 100     # variables is TV-biased
 101     keys = {'tvnn': 'callsign', 'tven': 'episodeNumber',
 102             'tvsh': 'seriesTitle'}
 103
 104     for key, value in mp4meta.items():
 105         if type(value) == list:
 106             value = value[0]
 107         if key == 'stik':
 108             metadata['isEpisode'] = ['false', 'true'][value == 'TV Show']
 109         elif key in keys:
 110             metadata[keys[key]] = value
 111         # These keys begin with the copyright symbol \xA9
 112         elif key == '\xa9day':
 113             if len(value) == 4:
 114                 value += '-01-01T16:00:00Z'
 115             metadata['originalAirDate'] = value
 116             #metadata['time'] = value
 117         elif key in ['\xa9gen', 'gnre']:
 118             for k in ('vProgramGenre', 'vSeriesGenre'):
 119                 if k in metadata:
 120                     metadata[k].append(value)
 121                 else:
 122                     metadata[k] = [value]
 123         elif key == '\xa9nam':
 124             if 'tvsh' in mp4meta:
 125                 metadata['episodeTitle'] = value
 126             else:
 127                 metadata['title'] = value
 128
 129         # Description in desc, cmt, and/or ldes tags. Keep the longest.
 130         elif key in ['desc', '\xa9cmt', 'ldes'] and len(value) > len_desc:
 131             metadata['description'] = value
 132             len_desc = len(value)
 133
 134         # A common custom "reverse DNS format" tag
 135         elif (key == '----:com.apple.iTunes:iTunEXTC' and
 136               ('us-tv' in value or 'mpaa' in value)):
 137             rating = value.split("|")[1].upper()
 138             if rating in TV_RATINGS and 'us-tv' in value:
 139                 metadata['tvRating'] = TV_RATINGS[rating]
 140             elif rating in MPAA_RATINGS and 'mpaa' in value:
 141                 metadata['mpaaRating'] = MPAA_RATINGS[rating]
 142
 143         # Actors, directors, producers, AND screenwriters may be in a long
 144         # embedded XML plist.
 145         elif (key == '----:com.apple.iTunes:iTunMOVI' and
 146               'plistlib' in sys.modules):
 147             items = {'cast': 'vActor', 'directors': 'vDirector',
 148                      'producers': 'vProducer', 'screenwriters': 'vWriter'}
 149             data = plistlib.readPlistFromString(value)
 150             for item in items:
 151                 if item in data:
 152                     metadata[items[item]] = [x['name'] for x in data[item]]
 153
 154     mp4_cache[full_path] = metadata
 155     return metadata
 156
 157 def from_mscore(rawmeta):
 158     metadata = {}
 159     keys = {'title': ['Title'],
 160             'description': ['Description', 'WM/SubTitleDescription'],
 161             'episodeTitle': ['WM/SubTitle'],
 162             'callsign': ['WM/MediaStationCallSign'],
 163             'displayMajorNumber': ['WM/MediaOriginalChannel'],
 164             'originalAirDate': ['WM/MediaOriginalBroadcastDateTime'],
 165             'rating': ['WM/ParentalRating'],
 166             'credits': ['WM/MediaCredits'], 'genre': ['WM/Genre']}
 167
 168     for tagname in keys:
 169         for tag in keys[tagname]:
 170             try:
 171                 if tag in rawmeta:
 172                     value = rawmeta[tag][0]
 173                     if type(value) not in (str, unicode):
 174                         value = str(value)
 175                     if value:
 176                         metadata[tagname] = value
 177             except:
 178                 pass
 179
 180     if 'episodeTitle' in metadata and 'title' in metadata:
 181         metadata['seriesTitle'] = metadata['title']
 182     if 'genre' in metadata:
 183         value = metadata['genre'].split(',')
 184         metadata['vProgramGenre'] = value
 185         metadata['vSeriesGenre'] = value
 186         del metadata['genre']
 187     if 'credits' in metadata:
 188         value = [x.split('/') for x in metadata['credits'].split(';')]
 189         if len(value) > 3:
 190             metadata['vActor'] = [x for x in (value[0] + value[3]) if x]
 191             metadata['vDirector'] = [x for x in value[1] if x]
 192         del metadata['credits']
 193     if 'rating' in metadata:
 194         rating = metadata['rating']
 195         if rating in TV_RATINGS:
 196             metadata['tvRating'] = TV_RATINGS[rating]
 197         del metadata['rating']
 198
 199     return metadata
 200
 201 def from_dvrms(full_path):
 202     if full_path in dvrms_cache:
 203         return dvrms_cache[full_path]
 204
 205     try:
 206         rawmeta = mutagen.File(unicode(full_path, 'utf-8'))
 207         assert(rawmeta)
 208     except:
 209         dvrms_cache[full_path] = {}
 210         return {}
 211
 212     metadata = from_mscore(rawmeta)
 213     dvrms_cache[full_path] = metadata
 214     return metadata
 215
 216 def from_eyetv(full_path):
 217     keys = {'TITLE': 'title', 'SUBTITLE': 'episodeTitle',
 218             'DESCRIPTION': 'description', 'YEAR': 'movieYear',
 219             'EPISODENUM': 'episodeNumber'}
 220     metadata = {}
 221     path, name = os.path.split(unicode(full_path, 'utf-8'))
 222     eyetvp = [x for x in os.listdir(path) if x.endswith('.eyetvp')][0]
 223     eyetvp = os.path.join(path, eyetvp)
 224     eyetv = plistlib.readPlist(eyetvp)
 225     if 'epg info' in eyetv:
 226         info = eyetv['epg info']
 227         for key in keys:
 228             if info[key]:
 229                 metadata[keys[key]] = info[key]
 230         if info['SUBTITLE']:
 231             metadata['seriesTitle'] = info['TITLE']
 232         if info['ACTORS']:
 233             metadata['vActor'] = [x.strip() for x in info['ACTORS'].split(',')]
 234         if info['DIRECTOR']:
 235             metadata['vDirector'] = [info['DIRECTOR']]
 236
 237         for ptag, etag, ratings in [('tvRating', 'TV_RATING', TV_RATINGS),
 238                               ('mpaaRating', 'MPAA_RATING', MPAA_RATINGS),
 239                               ('starRating', 'STAR_RATING', STAR_RATINGS)]:
 240            x = info[etag].upper()
 241            if x and x in ratings:
 242                metadata[ptag] = ratings[x]
 243
 244         # movieYear must be set for the mpaa/star ratings to work
 245         if (('mpaaRating' in metadata or 'starRating' in metadata) and
 246             'movieYear' not in metadata):
 247             metadata['movieYear'] = eyetv['info']['start'].year
 248     return metadata
 249
 250 def from_text(full_path):
 251     metadata = {}
 252     full_path = unicode(full_path, 'utf-8')
 253     path, name = os.path.split(full_path)
 254     title, ext = os.path.splitext(name)
 255
 256     for metafile in [os.path.join(path, title) + '.properties',
 257                      os.path.join(path, 'default.txt'), full_path + '.txt',
 258                      os.path.join(path, '.meta', 'default.txt'),
 259                      os.path.join(path, '.meta', name) + '.txt']:
 260         if os.path.exists(metafile):
 261             sep = ':='[metafile.endswith('.properties')]
 262             for line in file(metafile, 'U'):
 263                 if line.startswith(BOM):
 264                     line = line[3:]
 265                 if line.strip().startswith('#') or not sep in line:
 266                     continue
 267                 key, value = [x.strip() for x in line.split(sep, 1)]
 268                 if not key or not value:
 269                     continue
 270                 if key.startswith('v'):
 271                     if key in metadata:
 272                         metadata[key].append(value)
 273                     else:
 274                         metadata[key] = [value]
 275                 else:
 276                     metadata[key] = value
 277
 278     for rating, ratings in [('tvRating', TV_RATINGS),
 279                             ('mpaaRating', MPAA_RATINGS),
 280                             ('starRating', STAR_RATINGS)]:
 281         x = metadata.get(rating, '').upper()
 282         if x in ratings:
 283             metadata[rating] = ratings[x]
 284
 285     return metadata
 286
 287 def basic(full_path):
 288     base_path, name = os.path.split(full_path)
 289     title, ext = os.path.splitext(name)
 290     mtime = os.stat(unicode(full_path, 'utf-8')).st_mtime
 291     if (mtime < 0):
 292         mtime = 0
 293     originalAirDate = datetime.utcfromtimestamp(mtime)
 294
 295     metadata = {'title': title,
 296                 'originalAirDate': originalAirDate.isoformat()}
 297     ext = ext.lower()
 298     if ext in ['.mp4', '.m4v', '.mov']:
 299         metadata.update(from_moov(full_path))
 300     elif ext in ['.dvr-ms', '.asf', '.wmv']:
 301         metadata.update(from_dvrms(full_path))
 302     elif 'plistlib' in sys.modules and base_path.endswith('.eyetv'):
 303         metadata.update(from_eyetv(full_path))
 304     metadata.update(from_text(full_path))
 305
 306     return metadata
 307
 308 def from_container(xmldoc):
 309     metadata = {}
 310
 311     keys = {'title': 'Title', 'episodeTitle': 'EpisodeTitle',
 312             'description': 'Description', 'seriesId': 'SeriesId',
 313             'episodeNumber': 'EpisodeNumber', 'tvRating': 'TvRating',
 314             'displayMajorNumber': 'SourceChannel', 'callsign': 'SourceStation',
 315             'showingBits': 'ShowingBits', 'mpaaRating': 'MpaaRating'}
 316
 317     details = xmldoc.getElementsByTagName('Details')[0]
 318
 319     for key in keys:
 320         data = tag_data(details, keys[key])
 321         if data:
 322             if key == 'description':
 323                 data = data.replace(TRIBUNE_CR, '')
 324             elif key == 'tvRating':
 325                 data = int(data)
 326             elif key == 'displayMajorNumber':
 327                 if '-' in data:
 328                     data, metadata['displayMinorNumber'] = data.split('-')
 329             metadata[key] = data
 330
 331     return metadata
 332
 333 def from_details(xml):
 334     metadata = {}
 335
 336     xmldoc = minidom.parse(xml)
 337     showing = xmldoc.getElementsByTagName('showing')[0]
 338     program = showing.getElementsByTagName('program')[0]
 339
 340     items = {'description': 'program/description',
 341              'title': 'program/title',
 342              'episodeTitle': 'program/episodeTitle',
 343              'episodeNumber': 'program/episodeNumber',
 344              'seriesId': 'program/series/uniqueId',
 345              'seriesTitle': 'program/series/seriesTitle',
 346              'originalAirDate': 'program/originalAirDate',
 347              'isEpisode': 'program/isEpisode',
 348              'movieYear': 'program/movieYear',
 349              'partCount': 'partCount',
 350              'partIndex': 'partIndex',
 351              'time': 'time'}
 352
 353     for item in items:
 354         data = tag_data(showing, items[item])
 355         if data:
 356             if item == 'description':
 357                 data = data.replace(TRIBUNE_CR, '')
 358             metadata[item] = data
 359
 360     vItems = ['vActor', 'vChoreographer', 'vDirector',
 361               'vExecProducer', 'vProgramGenre', 'vGuestStar',
 362               'vHost', 'vProducer', 'vWriter']
 363
 364     for item in vItems:
 365         data = _vtag_data(program, item)
 366         if data:
 367             metadata[item] = data
 368
 369     sb = showing.getElementsByTagName('showingBits')
 370     if sb:
 371         metadata['showingBits'] = sb[0].attributes['value'].value
 372
 373     #for tag in ['starRating', 'mpaaRating', 'colorCode']:
 374     for tag in ['starRating', 'mpaaRating']:
 375         value = _tag_value(program, tag)
 376         if value:
 377             metadata[tag] = value
 378
 379     rating = _tag_value(showing, 'tvRating')
 380     if rating:
 381         metadata['tvRating'] = rating
 382
 383     return metadata
 384
 385 def from_tivo(full_path):
 386     if full_path in tivo_cache:
 387         return tivo_cache[full_path]
 388
 389     tdcat_path = config.get_bin('tdcat')
 390     tivo_mak = config.get_server('tivo_mak')
 391     try:
 392         assert(tdcat_path and tivo_mak)
 393         fname = unicode(full_path, 'utf-8')
 394         if mswindows:
 395             fname = fname.encode('iso8859-1')
 396         tcmd = [tdcat_path, '-m', tivo_mak, '-2', fname]
 397         tdcat = subprocess.Popen(tcmd, stdout=subprocess.PIPE)
 398         metadata = from_details(tdcat.stdout)
 399         tivo_cache[full_path] = metadata
 400     except:
 401         metadata = {}
 402
 403     return metadata
 404
 405 def force_utf8(text):
 406     if type(text) == str:
 407         try:
 408             text = text.decode('utf8')
 409         except:
 410             if sys.platform == 'darwin':
 411                 text = text.decode('macroman')
 412             else:
 413                 text = text.decode('iso8859-1')
 414     return text.encode('utf-8')
 415
 416 def dump(output, metadata):
 417     for key in metadata:
 418         value = metadata[key]
 419         if type(value) == list:
 420             for item in value:
 421                 output.write('%s: %s\n' % (key, item.encode('utf-8')))
 422         else:
 423             if key in HUMAN and value in HUMAN[key]:
 424                 output.write('%s: %s\n' % (key, HUMAN[key][value]))
 425             else:
 426                 output.write('%s: %s\n' % (key, value.encode('utf-8')))
 427
 428 if __name__ == '__main__':
 429     if len(sys.argv) > 1:
 430         metadata = {}
 431         fname = force_utf8(sys.argv[1])
 432         ext = os.path.splitext(fname)[1].lower()
 433         if ext == '.tivo':
 434             config.init([])
 435             metadata.update(from_tivo(fname))
 436         elif ext in ['.mp4', '.m4v', '.mov']:
 437             metadata.update(from_moov(fname))
 438         elif ext in ['.dvr-ms', '.asf', '.wmv']:
 439             metadata.update(from_dvrms(fname))
 440         elif ext == '.wtv':
 441             vInfo = plugins.video.transcode.video_info(fname)
 442             metadata.update(from_mscore(vInfo['rawmeta']))
 443         dump(sys.stdout, metadata)