audiomangler/codecs.py

   1 ###########################################################################
   2 #    Copyright (C) 2008 by Andrew Mahone
   3 #    <andrew.mahone@gmail.com>
   4 #
   5 # Copyright: See COPYING file that comes with this distribution
   6 #
   7 ###########################################################################
   8 import os, os.path
   9 import sys
  10 import re
  11 import Image
  12 try:
  13     from cStringIO import StringIO
  14 except ImportError:
  15     from StringIO import StringIO
  16 from tempfile import mkdtemp
  17 from threading import BoundedSemaphore, RLock
  18 from subprocess import Popen, PIPE
  19 from mutagen import FileType
  20 from audiomangler import Config, from_config, FuncTask, CLITask, TaskSet, Expr, Format, File, util, NormMetaData
  21
  22 class CodecMeta(type):
  23     def __new__(cls, name, bases, cls_dict):
  24         class_init = cls_dict.get('__classinit__',None)
  25         if class_init:
  26             cls_dict['__classinit__'] = staticmethod(class_init)
  27         return super(CodecMeta,cls).__new__(cls, name, bases, cls_dict)
  28
  29     def __init__(self, name, bases, cls_dict):
  30         if callable(getattr(self,'__classinit__',None)):
  31             self.__classinit__(self, name, bases, cls_dict)
  32
  33 codec_map = {}
  34
  35 class Codec(object):
  36     __metaclass__ = CodecMeta
  37
  38     _from_wav_multi = False
  39     _from_wav_pipe = False
  40     _to_wav_pipe = False
  41     _replaygain = False
  42     lossless = False
  43     def __classinit__(cls, name, bases, cls_dict):
  44         if 'type_' in cls_dict:
  45             codec_map[cls_dict['type_']] = cls
  46
  47     @classmethod
  48     def _conv_out_filename(cls, filename):
  49         return ''.join((filename.rsplit('.',1)[0],'.',cls.ext))
  50
  51     @classmethod
  52     def from_wav_multi(cls,indir,infiles,outfiles):
  53         if not getattr(cls,'_from_wav_multi_cmd',None):
  54             return None
  55         encopts = Config['encopts']
  56         if not encopts:
  57             encopts = ()
  58         else:
  59             encopts = tuple(encopts.split())
  60         args = cls._from_wav_multi_cmd.evaluate({
  61            'indir':indir,
  62            'infiles':tuple(infiles),
  63            'outfiles':tuple(outfiles),
  64            'ext':cls.ext,
  65            'type':cls.type_,
  66            'encopts':encopts,
  67            'encoder':cls.encoder
  68         })
  69         return (CLITask(args=args,stdin='/dev/null',stdout='/dev/null',stderr=sys.stderr,background=True))
  70
  71     @classmethod
  72     def from_wav_pipe(cls, infile, outfile):
  73         if not getattr(cls,'_from_wav_pipe_cmd',None):
  74             return None
  75         encopts = Config['encopts']
  76         if not encopts:
  77             encopts = ()
  78         else:
  79             encopts = tuple(encopts.split())
  80         outfile = cls._conv_out_filename(infile)
  81         env = {
  82             'infile': infile,
  83             'outfile': outfile,
  84             'ext': cls.ext,
  85             'type': cls.type_,
  86             'encopts': encopts,
  87             'encoder': cls.encoder
  88         }
  89         args = cls._from_wav_pipe_cmd.evaluate(env)
  90         stdin = '/dev/null'
  91         if hasattr(cls,'_from_wav_pipe_stdin'):
  92             stdin = cls._from_wav_pipe_stdin.evaluate(env)
  93         return CLITask(args=args,stdin=stdin,stdout='/dev/null',stderr=sys.stderr,background=True)
  94
  95     @classmethod
  96     def to_wav_pipe(cls, infile, outfile):
  97         if not getattr(cls,'_to_wav_pipe_cmd',None):
  98             return None
  99         env = {
 100            'infile': infile,
 101            'outfile': outfile,
 102            'ext':cls.ext,
 103            'type':cls.type_,
 104            'decoder':cls.decoder
 105         }
 106         args = cls._to_wav_pipe_cmd.evaluate(env)
 107         stdout = '/dev/null'
 108         if hasattr(cls,'_to_wav_pipe_stdout'):
 109             stdout = cls._to_wav_pipe_stdout.evaluate(env)
 110         return CLITask(args=args,stdin='/dev/null',stdout=stdout,stderr=sys.stderr,background=False)
 111
 112     @classmethod
 113     def add_replaygain(cls,files,metas=None):
 114         if metas and hasattr(cls, 'calc_replaygain'):
 115             tracks, album = cls.calc_replaygain(files)
 116             if tracks:
 117                 for meta,track in zip(metas,tracks):
 118                     meta.update(track)
 119                     meta.update(album)
 120             return metas
 121         elif hasattr(cls,'_replaygain_cmd'):
 122             env = {
 123             'replaygain':cls.replaygain,
 124             'files':tuple(files)
 125             }
 126             args = cls._replaygain_cmd.evaluate(env)
 127             CLITask(args=args,stdin='/dev/null',stdout='/dev/null',stderr='/dev/null',background=False).run()
 128         elif hasattr(cls,'calc_replaygain'):
 129             tracks, album = cls.calc_replaygain(files)
 130             for trackfile,trackgain in zip(files,tracks):
 131                 f = File(trackfile)
 132                 m = NormMetaData(trackgain + album)
 133                 m.update(trackgain)
 134                 m.update(album)
 135                 m.apply(f)
 136
 137
 138 class MP3Codec(Codec):
 139     ext = 'mp3'
 140     type_ = 'mp3'
 141     encoder = 'lame'
 142     replaygain = 'mp3gain'
 143     _from_wav_multi = True
 144     _replaygain = True
 145     _from_wav_multi_cmd = Expr("(encoder,'--quiet')+encopts+('--noreplaygain','--nogapout',indir,'--nogaptags','--nogap')+infiles")
 146
 147     @classmethod
 148     def calc_replaygain(cls,files):
 149         tracks = []
 150         args = [cls.replaygain,'-qo','-s','s']
 151         args.extend(files)
 152         p = Popen(args=args, stdout=PIPE, stderr=PIPE)
 153         (out, err) = p.communicate()
 154         out = [l.split('\t')[2:4] for l in out.splitlines()[1:]]
 155         for i in out[:-1]:
 156             gain = ' '.join((i[0],'dB'))
 157             peak = '%.8f'% (float(i[1]) / 32768)
 158             tracks.append((('replaygain_track_gain',gain),('replaygain_track_peak',peak)))
 159         gain = ' '.join((out[-1][0],'dB'))
 160         peak = '%.8f'% (float(out[-1][1]) / 32768)
 161         album = (('replaygain_album_gain',gain),('replaygain_album_peak',peak))
 162         return tracks,album
 163
 164 class WavPackCodec(Codec):
 165     ext = 'wv'
 166     type_ = 'wavpack'
 167     encoder = 'wavpack'
 168     decoder = 'wvunpack'
 169     replaygain = 'wvgain'
 170     _to_wav_pipe = True
 171     _from_wav_pipe = True
 172     _replaygain = True
 173     lossless = True
 174     _to_wav_pipe_cmd = Expr("(decoder,'-q','-w',infile,'-o','-')")
 175     _to_wav_pipe_stdout = Expr("outfile")
 176     _from_wav_pipe_cmd = Expr("(encoder,'-q')+encopts+(infile,'-o',outfile)")
 177     _replaygain_cmd = Expr("(replaygain,'-qa')+files")
 178
 179 class FLACCodec(Codec):
 180     ext = 'flac'
 181     type_ = 'flac'
 182     encoder = 'flac'
 183     decoder = 'flac'
 184     replaygain = 'metaflac'
 185     _to_wav_pipe = True
 186     _from_wav_pipe = True
 187     _replaygain = True
 188     lossless = True
 189     _to_wav_pipe_cmd = Expr("(decoder,'-s','-c','-d',infile)")
 190     _to_wav_pipe_stdout = Expr("outfile")
 191     _from_wav_pipe_cmd = Expr("(encoder,'-s')+encopts+(infile,)")
 192     _replaygain_cmd = Expr("(replaygain,'--add-replay-gain')+files")
 193
 194 class OggVorbisCodec(Codec):
 195     ext = 'ogg'
 196     type_ = 'oggvorbis'
 197     encoder = 'oggenc'
 198     decoder = 'oggdec'
 199     replaygain = 'vorbisgain'
 200     _to_wav_pipe = True
 201     _from_wav_pipe = True
 202     _replaygain = True
 203     _to_wav_pipe_cmd = Expr("(decoder,'-Q','-o','-',infile)")
 204     _to_wav_pipe_stdout = Expr("outfile")
 205     _from_wav_pipe_cmd = Expr("(encoder,'-Q')+encopts+('-o',outfile,infile)")
 206     _replaygain_cmd = Expr("(replaygain,'-q','-a')+files)")
 207
 208     @classmethod
 209     def calc_replaygain(cls,files):
 210         tracks = []
 211         args = [cls.replaygain,'-and']
 212         args.extend(files)
 213         p = Popen(args=args, stdout=PIPE, stderr=PIPE)
 214         (out, err) = p.communicate()
 215         apeak = 0.0
 216         for match in re.finditer('^\s*(\S+ dB)\s*\|\s*([0-9]+)\s*\|',out,re.M):
 217             gain = match.group(1)
 218             peak = float(match.group(2)) / 32768
 219             apeak = max(apeak,peak)
 220             peak = "%.8f" % peak
 221             tracks.append((('replaygain_track_gain',gain),('replaygain_track_peak',peak)))
 222         again = re.search('^Recommended Album Gain:\s*(\S+ dB)',err,re.M)
 223         if again:
 224             album = (('replaygain_album_gain',again.group(1)),('replaygain_album_peak',"%.8f" % apeak))
 225         else:
 226             album = (('replaygain_album_peak',apeak),)
 227         return tracks, album
 228
 229 def transcode_track(dtask, etask, sem):
 230     etask.run()
 231     dtask.run()
 232     etask.wait()
 233     if sem:
 234         sem.release()
 235
 236 def check_and_copy_cover(fileset,targetfiles):
 237     cover_sizes = Config['cover_sizes']
 238     if not cover_sizes:
 239         return
 240     cover_out_filename = Config['cover_out_filename']
 241     if not cover_out_filename:
 242         return
 243     cover_out_filename = Format(cover_out_filename)
 244     cover_sizes = cover_sizes.split(',')
 245     covers_loaded = {}
 246     covers_written = {}
 247     outdirs = set()
 248     cover_filenames = Config['cover_filenames']
 249     if cover_filenames:
 250         cover_filenames = cover_filenames.split(',')
 251     else:
 252         cover_filenames = ()
 253     cover_out_filenames = [cover_out_filename.evaluate({'size':s}) for s in cover_sizes]
 254     for (infile,targetfile) in zip(fileset,targetfiles):
 255         outdir = os.path.split(targetfile)[0]
 256         if outdir in outdirs: continue
 257         if reduce(lambda x,y: x and os.path.isfile(os.path.join(outdir,y)), cover_out_filenames, True):
 258             outdirs.add(outdir)
 259             continue
 260         i = None
 261         for filename in (os.path.join(infile.meta['dir'],f) for f in cover_filenames):
 262             try:
 263                 d = open(filename).read()
 264                 i = Image.open(StringIO(d))
 265                 i.load()
 266             except Exception:
 267                 continue
 268         if not i:
 269             tags = [(value.type,value) for key,value in infile.tags.items()
 270                 if key.startswith('APIC') and hasattr(value,'type')
 271                 and value.type in (0,3)]
 272             tags.sort(None,None,True)
 273             for t,value in tags:
 274                 i = None
 275                 try:
 276                     d = value.data
 277                     i = Image.open(StringIO(d))
 278                     i.load()
 279                     break
 280                 except Exception:
 281                     continue
 282         if not i: continue
 283         for s in cover_sizes:
 284             try:
 285                 s = int(s)
 286             except Exception:
 287                 continue
 288             w, h = i.size
 289             sc = 1.0*s/max(w,h)
 290             w = int(w*sc+0.5)
 291             h = int(h*sc+0.5)
 292             iw = i.resize((w,h),Image.ADAPTIVE)
 293             filename = os.path.join(
 294                outdir,cover_out_filename.evaluate({'size':s})
 295             )
 296             iw.save(filename)
 297         outdirs.add(outdir)
 298
 299 rg_keys = ('replaygain_track_gain','replaygain_track_peak','replaygain_album_gain','replaygain_album_peak')
 300 def transcode_set(targetcodec,fileset,targetfiles,alsem,trsem,workdirs,workdirs_l):
 301     try:
 302         if not fileset:
 303             workdirs_l = None
 304             return
 305         workdirs_l.acquire()
 306         workdir, pipefiles = workdirs.pop()
 307         workdirs_l.release()
 308         outfiles = map(targetcodec._conv_out_filename,pipefiles[:len(fileset)])
 309         if targetcodec._from_wav_pipe:
 310             for i,p,o in zip(fileset,pipefiles,outfiles):
 311                 bgprocs = set()
 312                 dtask = get_codec(i).to_wav_pipe(i.meta['path'],p)
 313                 etask = targetcodec.from_wav_pipe(p,o)
 314                 ttask = FuncTask(background=True,target=transcode_track,
 315                    args=(dtask,etask,trsem)
 316                 )
 317                 if trsem:
 318                     trsem.acquire()
 319                     bgprocs.add(ttask.run())
 320                 else:
 321                     ttask.runfg()
 322             for task in bgprocs:
 323                 task.wait()
 324         elif targetcodec._from_wav_multi:
 325             etask = targetcodec.from_wav_multi(
 326                workdir,pipefiles[:len(fileset)],outfiles
 327             )
 328             etask.run()
 329             for i,o in zip(fileset,pipefiles):
 330                 task = get_codec(i).to_wav_pipe(i.meta['path'],o)
 331                 task.run()
 332             etask.wait()
 333         dirs = set()
 334         metas = []
 335         newreplaygain = False
 336         for i,o in zip(fileset,outfiles):
 337             meta = i.meta.copy()
 338             if not (i.lossless and targetcodec.lossless):
 339                 for key in rg_keys:
 340                     if key in meta:
 341                         del meta[key]
 342                 newreplaygain = True
 343             if not newreplaygain:
 344                 for key in rg_keys:
 345                     if key not in meta:
 346                         newreplaygain=True
 347                         break
 348             metas.append(meta)
 349         if newreplaygain and targetcodec._replaygain:
 350             targetcodec.add_replaygain(outfiles,metas)
 351         for m,o,t in zip(metas,outfiles,targetfiles):
 352             o = File(o)
 353             m.apply(o)
 354             o.save()
 355             targetdir = os.path.split(t)[0]
 356             if targetdir not in dirs:
 357                 dirs.add(targetdir)
 358                 if not os.path.isdir(targetdir):
 359                     os.makedirs(targetdir)
 360             print "%s -> %s" %(i.filename,t)
 361             util.move(o.filename,t)
 362         check_and_copy_cover(fileset,targetfiles)
 363     finally:
 364         if workdirs_l:
 365             workdirs_l.acquire()
 366             workdirs.add((workdir,pipefiles))
 367             workdirs_l.release()
 368         if alsem:
 369             alsem.release()
 370
 371 def sync_sets(sets=[],targettids=()):
 372     try:
 373         semct = int(Config['jobs'])
 374     except (ValueError,TypeError):
 375         semct = 1
 376     bgtasks = set()
 377     targetcodec = Config['type']
 378     if ',' in targetcodec:
 379         allowedcodecs = targetcodec.split(',')
 380         targetcodec = allowedcodecs[0]
 381         allowedcodecs = set(allowedcodecs)
 382     else:
 383         allowedcodecs = set((targetcodec,))
 384     targetcodec = get_codec(targetcodec)
 385     workdir = Config['workdir'] or Config['base']
 386     workdir = mkdtemp(dir=workdir,prefix='audiomangler_work_')
 387     if targetcodec._from_wav_pipe:
 388         if len(sets) > semct * 2:
 389             alsem = BoundedSemaphore(semct)
 390             trsem = None
 391         else:
 392             trsem = BoundedSemaphore(semct)
 393             alsem = None
 394     elif targetcodec._from_wav_multi:
 395         trsem = None
 396         alsem = BoundedSemaphore(semct)
 397     numpipes = max(len(s) for s in sets)
 398     workdirs = set()
 399     workdirs_l = RLock()
 400     for n in range(semct):
 401         w = os.path.join(workdir,"%02d" % n)
 402         os.mkdir(w)
 403         pipes = []
 404         for m in range(numpipes):
 405             pipes.append(os.path.join(w,"%02d.wav"%m))
 406             os.mkfifo(pipes[-1])
 407         pipes = tuple(pipes)
 408         workdirs.add((w,pipes))
 409     for fileset in sets:
 410         if reduce(lambda x,y: x and (y.type_ in allowedcodecs), fileset, True):
 411             targetfiles = [f.format() for f in fileset]
 412             if not reduce(lambda x,y: x and (y.tid in targettids),
 413                fileset, True):
 414                 print "copying files"
 415                 dirs = set()
 416                 for i in fileset:
 417                     t = i.format()
 418                     targetdir = os.path.split(t)[0]
 419                     if targetdir not in dirs:
 420                         dirs.add(targetdir)
 421                         if not os.path.isdir(targetdir):
 422                             os.makedirs(targetdir)
 423                     print "%s -> %s" % (i.filename,t)
 424                     util.copy(i.filename,t)
 425             continue
 426         if alsem:
 427             alsem.acquire()
 428             for task in list(bgtasks):
 429                 if task.poll():
 430                     bgtasks.remove(task)
 431         postadd = {'type':targetcodec.type_,'ext':targetcodec.ext}
 432         targetfiles = [f.format(postadd=postadd)for f in fileset]
 433         task = FuncTask(
 434            background=True,target=transcode_set,args=(
 435               targetcodec,fileset,targetfiles,alsem,trsem,workdirs,workdirs_l
 436         ))
 437         if alsem:
 438             bgtasks.add(task.run())
 439         else:
 440             task.runfg()
 441     for task in bgtasks:
 442         task.wait()
 443     for w,ps in workdirs:
 444         for p in ps:
 445             os.unlink(p)
 446         os.rmdir(w)
 447     os.rmdir(workdir)
 448
 449 def get_codec(item):
 450     if isinstance(item, FileType):
 451         item = getattr(item,'type_')
 452     return codec_map[item]
 453
 454 __all__ = ['sync_sets','get_codec']