metadata.py

   1 #!/usr/bin/env python
   2
   3 import os
   4 import subprocess
   5 import sys
   6 from datetime import datetime
   7 from xml.dom import minidom
   8 try:
   9     import plistlib
  10 except:
  11     pass
  12
  13 import mutagen
  14 from lrucache import LRUCache
  15
  16 import config
  17
  18 # Something to strip
  19 TRIBUNE_CR = ' Copyright Tribune Media Services, Inc.'
  20
  21 TV_RATINGS = {'TV-Y7': 'x1', 'TV-Y': 'x2', 'TV-G': 'x3', 'TV-PG': 'x4',
  22               'TV-14': 'x5', 'TV-MA': 'x6', 'TV-NR': 'x7',
  23               'TVY7': 'x1', 'TVY': 'x2', 'TVG': 'x3', 'TVPG': 'x4',
  24               'TV14': 'x5', 'TVMA': 'x6', 'TVNR': 'x7',
  25               'Y7': 'x1', 'Y': 'x2', 'G': 'x3', 'PG': 'x4',
  26               '14': 'x5', 'MA': 'x6', 'NR': 'x7', 'UNRATED': 'x7'}
  27
  28 MPAA_RATINGS = {'G': 'G1', 'PG': 'P2', 'PG-13': 'P3', 'PG13': 'P3',
  29                 'R': 'R4', 'X': 'X5', 'NC-17': 'N6', 'NC17': 'N6',
  30                 'NR': 'N8', 'UNRATED': 'N8'}
  31
  32 STAR_RATINGS = {'1': 'x1', '1.5': 'x2', '2': 'x3', '2.5': 'x4',
  33                 '3': 'x5', '3.5': 'x6', '4': 'x7',
  34                 '*': 'x1', '**': 'x3', '***': 'x5', '****': 'x7'}
  35
  36 HUMAN = {'mpaaRating': {'G1': 'G', 'P2': 'PG', 'P3': 'PG-13', 'R4': 'R',
  37                         'X5': 'X', 'N6': 'NC-17', 'N8': 'Unrated'},
  38          'tvRating': {'x1': 'TV-Y7', 'x2': 'TV-Y', 'x3': 'TV-G',
  39                       'x4': 'TV-PG', 'x5': 'TV-14', 'x6': 'TV-MA',
  40                       'x7': 'Unrated'},
  41          'starRating': {'x1': '1', 'x2': '1.5', 'x3': '2', 'x4': '2.5',
  42                         'x5': '3', 'x6': '3.5', 'x7': '4'}}
  43
  44 BOM = '\xef\xbb\xbf'
  45
  46 tivo_cache = LRUCache(50)
  47 mp4_cache = LRUCache(50)
  48 dvrms_cache = LRUCache(50)
  49
  50 mswindows = (sys.platform == "win32")
  51
  52 def tag_data(element, tag):
  53     for name in tag.split('/'):
  54         new_element = element.getElementsByTagName(name)
  55         if not new_element:
  56             return ''
  57         element = new_element[0]
  58     if not element.firstChild:
  59         return ''
  60     return element.firstChild.data
  61
  62 def _vtag_data(element, tag):
  63     for name in tag.split('/'):
  64         new_element = element.getElementsByTagName(name)
  65         if not new_element:
  66             return []
  67         element = new_element[0]
  68     elements = element.getElementsByTagName('element')
  69     return [x.firstChild.data for x in elements if x.firstChild]
  70
  71 def _tag_value(element, tag):
  72     item = element.getElementsByTagName(tag)
  73     if item:
  74         value = item[0].attributes['value'].value
  75         name = item[0].firstChild.data
  76         return name[0] + value[0]
  77
  78 def from_moov(full_path):
  79     if full_path in mp4_cache:
  80         return mp4_cache[full_path]
  81
  82     metadata = {}
  83     len_desc = 0
  84
  85     try:
  86         mp4meta = mutagen.File(unicode(full_path, 'utf-8'))
  87         assert(mp4meta)
  88     except:
  89         mp4_cache[full_path] = {}
  90         return {}
  91
  92     # The following 1-to-1 correspondence of atoms to pyTivo
  93     # variables is TV-biased
  94     keys = {'tvnn': 'callsign', 'tven': 'episodeNumber',
  95             'tvsh': 'seriesTitle'}
  96
  97     for key, value in mp4meta.items():
  98         if type(value) == list:
  99             value = value[0]
 100         if key == 'stik':
 101             metadata['isEpisode'] = ['false', 'true'][value == 'TV Show']
 102         elif key in keys:
 103             metadata[keys[key]] = value
 104         # These keys begin with the copyright symbol \xA9
 105         elif key == '\xa9day':
 106             if len(value) == 4:
 107                 value += '-01-01T16:00:00Z'
 108             metadata['originalAirDate'] = value
 109             #metadata['time'] = value
 110         elif key in ['\xa9gen', 'gnre']:
 111             for k in ('vProgramGenre', 'vSeriesGenre'):
 112                 if k in metadata:
 113                     metadata[k].append(value)
 114                 else:
 115                     metadata[k] = [value]
 116         elif key == '\xa9nam':
 117             if 'tvsh' in mp4meta:
 118                 metadata['episodeTitle'] = value
 119             else:
 120                 metadata['title'] = value
 121
 122         # Description in desc, cmt, and/or ldes tags. Keep the longest.
 123         elif key in ['desc', '\xa9cmt', 'ldes'] and len(value) > len_desc:
 124             metadata['description'] = value
 125             len_desc = len(value)
 126
 127         # A common custom "reverse DNS format" tag
 128         elif (key == '----:com.apple.iTunes:iTunEXTC' and
 129               ('us-tv' in value or 'mpaa' in value)):
 130             rating = value.split("|")[1].upper()
 131             if rating in TV_RATINGS and 'us-tv' in value:
 132                 metadata['tvRating'] = TV_RATINGS[rating]
 133             elif rating in MPAA_RATINGS and 'mpaa' in value:
 134                 metadata['mpaaRating'] = MPAA_RATINGS[rating]
 135
 136         # Actors, directors, producers, AND screenwriters may be in a long
 137         # embedded XML plist.
 138         elif (key == '----:com.apple.iTunes:iTunMOVI' and
 139               'plistlib' in sys.modules):
 140             items = {'cast': 'vActor', 'directors': 'vDirector',
 141                      'producers': 'vProducer', 'screenwriters': 'vWriter'}
 142             data = plistlib.readPlistFromString(value)
 143             for item in items:
 144                 if item in data:
 145                     metadata[items[item]] = [x['name'] for x in data[item]]
 146
 147     mp4_cache[full_path] = metadata
 148     return metadata
 149
 150 def from_dvrms(full_path):
 151     if full_path in dvrms_cache:
 152         return dvrms_cache[full_path]
 153
 154     metadata = {}
 155
 156     try:
 157         meta = mutagen.File(unicode(full_path, 'utf-8'))
 158         assert(meta)
 159     except:
 160         dvrms_cache[full_path] = {}
 161         return {}
 162
 163     keys = {'title': ['Title'],
 164             'description': ['Description', 'WM/SubTitleDescription'],
 165             'episodeTitle': ['WM/SubTitle'],
 166             'callsign': ['WM/MediaStationCallSign'],
 167             'displayMajorNumber': ['WM/MediaOriginalChannel'],
 168             'originalAirDate': ['WM/MediaOriginalBroadcastDateTime'],
 169             'rating': ['WM/ParentalRating'],
 170             'credits': ['WM/MediaCredits'], 'genre': ['WM/Genre']}
 171
 172     for tagname in keys:
 173         for tag in keys[tagname]:
 174             try:
 175                 if tag in meta:
 176                     value = str(meta[tag][0])
 177                     if value:
 178                         metadata[tagname] = value
 179             except:
 180                 pass
 181
 182     if 'episodeTitle' in metadata and 'title' in metadata:
 183         metadata['seriesTitle'] = metadata['title']
 184     if 'genre' in metadata:
 185         value = metadata['genre'].split(',')
 186         metadata['vProgramGenre'] = value
 187         metadata['vSeriesGenre'] = value
 188         del metadata['genre']
 189     if 'credits' in metadata:
 190         value = [x.split('/') for x in metadata['credits'].split(';')]
 191         if len(value) > 3:
 192             metadata['vActor'] = [x for x in (value[0] + value[3]) if x]
 193             metadata['vDirector'] = [x for x in value[1] if x]
 194         del metadata['credits']
 195     if 'rating' in metadata:
 196         rating = metadata['rating']
 197         if rating in TV_RATINGS:
 198             metadata['tvRating'] = TV_RATINGS[rating]
 199         del metadata['rating']
 200
 201     dvrms_cache[full_path] = metadata
 202     return metadata
 203
 204 def from_eyetv(full_path):
 205     keys = {'TITLE': 'title', 'SUBTITLE': 'episodeTitle',
 206             'DESCRIPTION': 'description', 'YEAR': 'movieYear',
 207             'EPISODENUM': 'episodeNumber'}
 208     metadata = {}
 209     path, name = os.path.split(unicode(full_path, 'utf-8'))
 210     eyetvp = [x for x in os.listdir(path) if x.endswith('.eyetvp')][0]
 211     eyetvp = os.path.join(path, eyetvp)
 212     eyetv = plistlib.readPlist(eyetvp)
 213     if 'epg info' in eyetv:
 214         info = eyetv['epg info']
 215         for key in keys:
 216             if info[key]:
 217                 metadata[keys[key]] = info[key]
 218         if info['SUBTITLE']:
 219             metadata['seriesTitle'] = info['TITLE']
 220         if info['ACTORS']:
 221             metadata['vActor'] = [x.strip() for x in info['ACTORS'].split(',')]
 222         if info['DIRECTOR']:
 223             metadata['vDirector'] = [info['DIRECTOR']]
 224
 225         for ptag, etag, ratings in [('tvRating', 'TV_RATING', TV_RATINGS),
 226                               ('mpaaRating', 'MPAA_RATING', MPAA_RATINGS),
 227                               ('starRating', 'STAR_RATING', STAR_RATINGS)]:
 228            x = info[etag].upper()
 229            if x and x in ratings:
 230                metadata[ptag] = ratings[x]
 231
 232         # movieYear must be set for the mpaa/star ratings to work
 233         if (('mpaaRating' in metadata or 'starRating' in metadata) and
 234             'movieYear' not in metadata):
 235             metadata['movieYear'] = eyetv['info']['start'].year
 236     return metadata
 237
 238 def from_text(full_path):
 239     metadata = {}
 240     full_path = unicode(full_path, 'utf-8')
 241     path, name = os.path.split(full_path)
 242     title, ext = os.path.splitext(name)
 243
 244     for metafile in [os.path.join(path, title) + '.properties',
 245                      os.path.join(path, 'default.txt'), full_path + '.txt',
 246                      os.path.join(path, '.meta', 'default.txt'),
 247                      os.path.join(path, '.meta', name) + '.txt']:
 248         if os.path.exists(metafile):
 249             sep = ':='[metafile.endswith('.properties')]
 250             for line in file(metafile, 'U'):
 251                 if line.startswith(BOM):
 252                     line = line[3:]
 253                 if line.strip().startswith('#') or not sep in line:
 254                     continue
 255                 key, value = [x.strip() for x in line.split(sep, 1)]
 256                 if not key or not value:
 257                     continue
 258                 if key.startswith('v'):
 259                     if key in metadata:
 260                         metadata[key].append(value)
 261                     else:
 262                         metadata[key] = [value]
 263                 else:
 264                     metadata[key] = value
 265
 266     for rating, ratings in [('tvRating', TV_RATINGS),
 267                             ('mpaaRating', MPAA_RATINGS),
 268                             ('starRating', STAR_RATINGS)]:
 269         x = metadata.get(rating, '').upper()
 270         if x in ratings:
 271             metadata[rating] = ratings[x]
 272
 273     return metadata
 274
 275 def basic(full_path):
 276     base_path, name = os.path.split(full_path)
 277     title, ext = os.path.splitext(name)
 278     mtime = os.stat(unicode(full_path, 'utf-8')).st_mtime
 279     if (mtime < 0):
 280         mtime = 0
 281     originalAirDate = datetime.fromtimestamp(mtime)
 282
 283     metadata = {'title': title,
 284                 'originalAirDate': originalAirDate.isoformat()}
 285     ext = ext.lower()
 286     if ext in ['.mp4', '.m4v', '.mov']:
 287         metadata.update(from_moov(full_path))
 288     elif ext in ['.dvr-ms', '.asf', '.wmv']:
 289         metadata.update(from_dvrms(full_path))
 290     elif 'plistlib' in sys.modules and base_path.endswith('.eyetv'):
 291         metadata.update(from_eyetv(full_path))
 292     metadata.update(from_text(full_path))
 293
 294     return metadata
 295
 296 def from_container(xmldoc):
 297     metadata = {}
 298
 299     keys = {'title': 'Title', 'episodeTitle': 'EpisodeTitle',
 300             'description': 'Description', 'seriesId': 'SeriesId',
 301             'episodeNumber': 'EpisodeNumber', 'tvRating': 'TvRating',
 302             'displayMajorNumber': 'SourceChannel', 'callsign': 'SourceStation'}
 303
 304     details = xmldoc.getElementsByTagName('Details')[0]
 305
 306     for key in keys:
 307         data = tag_data(details, keys[key])
 308         if data:
 309             if key == 'description':
 310                 data = data.replace(TRIBUNE_CR, '')
 311             elif key == 'tvRating':
 312                 data = 'x' + data
 313             elif key == 'displayMajorNumber':
 314                 if '-' in data:
 315                     data, metadata['displayMinorNumber'] = data.split('-')
 316             metadata[key] = data
 317
 318     return metadata
 319
 320 def from_details(xml):
 321     metadata = {}
 322
 323     xmldoc = minidom.parse(xml)
 324     showing = xmldoc.getElementsByTagName('showing')[0]
 325     program = showing.getElementsByTagName('program')[0]
 326
 327     items = {'description': 'program/description',
 328              'title': 'program/title',
 329              'episodeTitle': 'program/episodeTitle',
 330              'episodeNumber': 'program/episodeNumber',
 331              'seriesId': 'program/series/uniqueId',
 332              'seriesTitle': 'program/series/seriesTitle',
 333              'originalAirDate': 'program/originalAirDate',
 334              'isEpisode': 'program/isEpisode',
 335              'movieYear': 'program/movieYear',
 336              'partCount': 'partCount',
 337              'partIndex': 'partIndex',
 338              'time': 'time'}
 339
 340     for item in items:
 341         data = tag_data(showing, items[item])
 342         if data:
 343             if item == 'description':
 344                 data = data.replace(TRIBUNE_CR, '')
 345             metadata[item] = data
 346
 347     vItems = ['vActor', 'vChoreographer', 'vDirector',
 348               'vExecProducer', 'vProgramGenre', 'vGuestStar',
 349               'vHost', 'vProducer', 'vWriter']
 350
 351     for item in vItems:
 352         data = _vtag_data(program, item)
 353         if data:
 354             metadata[item] = data
 355
 356     sb = showing.getElementsByTagName('showingBits')
 357     if sb:
 358         metadata['showingBits'] = sb[0].attributes['value'].value
 359
 360     for tag in ['starRating', 'mpaaRating', 'colorCode']:
 361         value = _tag_value(program, tag)
 362         if value:
 363             metadata[tag] = value
 364
 365     rating = _tag_value(showing, 'tvRating')
 366     if rating:
 367         metadata['tvRating'] = 'x' + rating[1]
 368
 369     return metadata
 370
 371 def from_tivo(full_path):
 372     if full_path in tivo_cache:
 373         return tivo_cache[full_path]
 374
 375     tdcat_path = config.get_bin('tdcat')
 376     tivo_mak = config.get_server('tivo_mak')
 377     if tdcat_path and tivo_mak:
 378         fname = unicode(full_path, 'utf-8')
 379         if mswindows:
 380             fname = fname.encode('iso8859-1')
 381         tcmd = [tdcat_path, '-m', tivo_mak, '-2', fname]
 382         tdcat = subprocess.Popen(tcmd, stdout=subprocess.PIPE)
 383         metadata = from_details(tdcat.stdout)
 384         tivo_cache[full_path] = metadata
 385     else:
 386         metadata = {}
 387
 388     return metadata
 389
 390 def force_utf8(text):
 391     if type(text) == str:
 392         try:
 393             text = text.decode('utf8')
 394         except:
 395             if sys.platform == 'darwin':
 396                 text = text.decode('macroman')
 397             else:
 398                 text = text.decode('iso8859-1')
 399     return text.encode('utf-8')
 400
 401 def dump(output, metadata):
 402     for key in metadata:
 403         value = metadata[key]
 404         if type(value) == list:
 405             for item in value:
 406                 output.write('%s: %s\n' % (key, item.encode('utf-8')))
 407         else:
 408             if key in HUMAN and value in HUMAN[key]:
 409                 output.write('%s: %s\n' % (key, HUMAN[key][value]))
 410             else:
 411                 output.write('%s: %s\n' % (key, value.encode('utf-8')))
 412
 413 if __name__ == '__main__':
 414     if len(sys.argv) > 1:
 415         metadata = {}
 416         fname = force_utf8(sys.argv[1])
 417         ext = os.path.splitext(fname)[1].lower()
 418         if ext == '.tivo':
 419             config.init([])
 420             metadata.update(from_tivo(fname))
 421         elif ext in ['.mp4', '.m4v', '.mov']:
 422             metadata.update(from_moov(fname))
 423         elif ext in ['.dvr-ms', '.asf', '.wmv']:
 424             metadata.update(from_dvrms(fname))
 425         dump(sys.stdout, metadata)