From d4a8811be6bbcf4dd112270193d04594c42da7c8 Mon Sep 17 00:00:00 2001 From: Andrew Mahone Date: Fri, 2 Jul 2010 23:17:17 -0400 Subject: [PATCH] add logging module, use logging in cli.rename, cli uses decorator to parse options, expression/format fixes, begin task rework (tasks broken now) --- audiomangler/__init__.py | 8 ++- audiomangler/cli.py | 114 +++++++++++++++++++++--------------- audiomangler/config.py | 10 +++- audiomangler/expression.py | 2 +- audiomangler/logging.py | 143 +++++++++++++++++++++++++++++++++++++++++++++ audiomangler/scanner.py | 2 +- audiomangler/task.py | 90 ++++++++++------------------ audiomangler/util.py | 36 +++++++++++- 8 files changed, 294 insertions(+), 111 deletions(-) create mode 100644 audiomangler/logging.py diff --git a/audiomangler/__init__.py b/audiomangler/__init__.py index b322aac..623c06c 100644 --- a/audiomangler/__init__.py +++ b/audiomangler/__init__.py @@ -7,18 +7,22 @@ # ########################################################################### from mutagen import File -from audiomangler import util from audiomangler.config import * +from audiomangler.logging import * +from audiomangler import util +from audiomangler.task import * from audiomangler.expression import * from audiomangler.tag import * from audiomangler.scanner import * -from audiomangler.task import * from audiomangler.codecs import * from audiomangler import mutagenext from audiomangler.cli import * + + __all__ = [ 'File', 'Format', + 'FileFormat' 'Expr', 'Evaluate', 'NormMetaData', diff --git a/audiomangler/cli.py b/audiomangler/cli.py index 5aa07ee..8bd74b2 100644 --- a/audiomangler/cli.py +++ b/audiomangler/cli.py @@ -12,34 +12,41 @@ import shutil import os import os.path import errno -from audiomangler import scan, Config, util, sync_sets, get_codec +from functools import wraps +from audiomangler import scan, Config, util, sync_sets, get_codec, err, msg, fatal, ERROR, WARNING, INFO, DEBUG -def parse_options(args = None, options = []): - if args is None and len(sys.argv) == 1: - print_usage(options) - sys.exit(0) - if args == None: - args = sys.argv[1:] - name_map = {} - s_opts = [] - l_opts = [] - for (s_opt, l_opt, name, desc) in options: - if s_opt: - name_map['-'+s_opt.rstrip(':')] = name - s_opts.append(s_opt) - if l_opt: - name_map['--'+l_opt.rstrip('=')] = name - l_opts.append(l_opt) - s_opts = ''.join(s_opts) - try: - (opts, args) = getopt.getopt(args,s_opts,l_opts) - except getopt.GetoptError: - print_usage(options) - sys.exit(0) - for k,v in opts: - k = name_map[k] - Config[k] = v - return args +def parse_options(options = []): + def decorator(f): + @wraps(f) + def proxy(*args): + if not args: + if len(sys.argv) == 1: + print_usage(options) + sys.exit(0) + else: + args = sys.argv[1:] + name_map = {} + s_opts = [] + l_opts = [] + for (s_opt, l_opt, name, desc) in options: + if s_opt: + name_map['-'+s_opt.rstrip(':')] = name + s_opts.append(s_opt) + if l_opt: + name_map['--'+l_opt.rstrip('=')] = name + l_opts.append(l_opt) + s_opts = ''.join(s_opts) + try: + (opts, args) = getopt.getopt(args,s_opts,l_opts) + except getopt.GetoptError: + print_usage(options) + sys.exit(0) + for k,v in opts: + k = name_map[k] + Config[k] = v + f(*args) + return proxy + return decorator def print_usage(opts): print """usage: @@ -55,22 +62,26 @@ common_opts = ( ('f:','filename=','filename','format for target filenames'), ) -rename_opts = common_opts -def rename(args = None): - args = parse_options(args, rename_opts) +@parse_options(common_opts) +def rename(*args): dir_list = scan(args)[1] + util.test_splits(dir_list) + onsplit = Config['onsplit'] for (dir_,files) in dir_list.items(): - dir_ = dir_.encode(Config['fs_encoding'],Config['fs_encoding_error'] or 'replace') - print "from dir %s:" % dir_ + dir_p = util.fsdecode(dir_) + msg(consoleformat=u"from dir %(dir_p)s:", + format="enter: %(dir_)r", dir_=dir_, dir_p=dir_p, loglevel=INFO) dstdirs = set() moves = [] for file_ in files: src = file_.filename - dst = file_.format() + dst = util.fsencode(file_.format()) + src_p = util.fsdecode(src) + dst_p = util.fsdecode(dst) if src == dst: - print " skipping %s, already named correctly" % src + msg(consoleformat=u" skipping %(src_p)s, already named correctly", + format="skip: %(src)r", src_p=srcp_p, src=src, loglevel=INFO) continue - print " %s -> %s" % (src, dst) dstdir = os.path.split(dst)[0] if dstdir not in dstdirs and dstdir != dir_: try: @@ -79,16 +90,23 @@ def rename(args = None): if e.errno != errno.EEXIST or not os.path.isdir(dstdir): raise dstdirs.add(dstdir) + msg(consoleformat=u" %(src_p)s -> %(dst_p)s", + format="move: %(src)r, %(dst)r", src=src, dst=dst, src_p=src_p, dst_p=dst_p, loglevel=INFO) util.move(src,dst) if len(dstdirs) == 1: dstdir = dstdirs.pop() for file_ in os.listdir(dir_): src = os.path.join(dir_,file_) dst = os.path.join(dstdir,file_) - print " %s -> %s" % (src,dst) + src_p = util.fsdecode(src) + dst_p = util.fsdecode(dst) + msg(consoleformat=u" %(src_p)s -> %(dst_p)s", + format="move: %(src)r, %(dst)r", src=src, dst=dst, src_p=src_p, dst_p=dst_p, loglevel=INFO) util.move(src,dst) while len(os.listdir(dir_)) == 0: - print " removing empty directory: %s" % dir_ + dir_p = util.fsdecode(dir_) + msg(consoleformat=u" remove empty directory: %(dir_p)s", + format="rmdir: %(dir_)r", dir_=dir_, dir_p=dir_p, loglevel=INFO) try: os.rmdir(dir_) except Exception: @@ -98,20 +116,24 @@ def rename(args = None): dir_ = newdir else: break + else: + if onsplit == 'warn': + msg(consoleformat=u"WARNING: tracks in %(dir_p)s were placed in different directories, other files may be left in the source directory", + format="split: %(dir_)r", dir_=dir_, dir_p=dir_p, loglevel=WARNING) -sync_opts = common_opts + ( - ('t:','type=','type','type of audio to encode to'), - ('s:','preset=','preset','codec preset to use'), - ('e:','encopts=','encopts','encoder options to use'), - ('j:','jobs=','jobs','number of jobs to run'), +@parse_options(common_opts + ( + ('t:','type=','type','type of audio to encode to'), + ('s:','preset=','preset','codec preset to use'), + ('e:','encopts=','encopts','encoder options to use'), + ('j:','jobs=','jobs','number of jobs to run'), + ) ) -def sync(args = None): - args = parse_options(args, sync_opts) +def sync(*args): (album_list, dir_list) = scan(args)[:2] targettids = scan(Config['base'])[2] sync_sets(album_list.values(),targettids) -replaygain_opts = common_opts[:2] +@parse_options(common_opts[:2]) def replaygain(args = None): args = parse_options(args, sync_opts) if not args: @@ -137,4 +159,4 @@ def replaygain(args = None): continue codec.add_replaygain([t.filename for t in album]) -__all__ = ['rename'] \ No newline at end of file +__all__ = [] \ No newline at end of file diff --git a/audiomangler/config.py b/audiomangler/config.py index c4f440a..f04cb05 100644 --- a/audiomangler/config.py +++ b/audiomangler/config.py @@ -64,7 +64,7 @@ class AMConfig(RawConfigParser): if not source: continue try: - ret = self.get(source,key) + ret = RawConfigParser.get(self,source,key) except (NoOptionError, NoSectionError): continue else: @@ -72,12 +72,18 @@ class AMConfig(RawConfigParser): return ret self._cache[key] = None + def get(self, key, default=None): + ret = self[key] + return ret if ret is not None else default + + for n in ('read','readfp','remove_option','remove_section','set'): locals()[n] = clear_cache(getattr(RawConfigParser,n)) Config = AMConfig( ( ('defaults', + ('onsplit', 'abort'), ('groupby', "first(" "('album',musicbrainz_albumid)," @@ -85,6 +91,8 @@ Config = AMConfig( "('dir',dir)" ")" ), + ('loglevel','INFO'), + ('consolelevel','INFO'), ('trackid', "first(" "('mbid',musicip_puid,musicbrainz_albumid,first(tracknumber))," diff --git a/audiomangler/expression.py b/audiomangler/expression.py index 39d47d6..3907a42 100644 --- a/audiomangler/expression.py +++ b/audiomangler/expression.py @@ -240,7 +240,7 @@ class FileFormat(SanitizedFormat): def evaluate(self, cdict): ret = super(self.__class__,self).evaluate(cdict) if ret is not None: - ret = ret.translate(pathtrans).encode(Config['fs_encoding'],Config['fs_encoding_err'] or 'underscorereplace') + ret = ret.translate(pathtrans) return ret #class Format(Expr): diff --git a/audiomangler/logging.py b/audiomangler/logging.py new file mode 100644 index 0000000..275512f --- /dev/null +++ b/audiomangler/logging.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- +from twisted.python import log, failure +from audiomangler import Config +import os, sys, atexit + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + +loglevels = dict(ERROR=0, WARNING=1, INFO=2, DEBUG=3) +locals().update(loglevels) +loglevels.update(map(reversed, loglevels.items())) + +class FilteredFileLogObserver(log.FileLogObserver): + def __init__(self, f, loglevel=INFO): + self.output = f + if isinstance(loglevel, basestring): + loglevel = loglevels[loglevel] + self.loglevel = loglevel + + def emit(self, eventDict): + if eventDict.get('loglevel',DEBUG) > self.loglevel: + return + encoding = sys.stdout.encoding + if eventDict['isError'] and 'failure' in eventDict: + text = log.textFromEventDict(eventDict) + elif eventDict['message']: + text = ' '.join(s.encode(encoding,'replace') if isinstance(s,unicode) else s for s in eventDict['message']) + elif 'format' in eventDict or 'consoleformat' in eventDict: + fmt = eventDict.get('format',eventDict.get('consoleformat')) + text = (fmt % eventDict).encode(encoding,'replace') + else: + text = log.textFromEventDict(eventDict) + timeStr = self.formatTime(eventDict['time']) + self.output.write(timeStr + ' [' + loglevels[eventDict['loglevel']].ljust(7) + '] ' + text + '\n') + +class FilteredConsoleLogObserver: + def __init__(self, loglevel=INFO): + self.loglevel = loglevel + + def start(self): + log.addObserver(self.emit) + + def stop(self): + log.removeObserver(self.emit) + + def emit(self, eventDict): + if eventDict.get('loglevel',DEBUG) > self.loglevel: + return + encoding = sys.stdout.encoding + if eventDict['isError'] and 'failure' in eventDict: + text = log.textFromEventDict(eventDict) + elif eventDict['message']: + text = ''.join(s.encode(encoding,'replace') if isinstance(s,unicode) else s for s in eventDict['message']) + elif 'format' in eventDict or 'consoleformat' in eventDict: + fmt = eventDict.get('consoleformat',eventDict.get('format')) + text = (fmt % eventDict).encode(encoding,'replace') + else: + text = log.textFromEventDict(eventDict) + sys.stdout.write(text + '\n') + sys.stdout.flush() + +collector = None +logfile = None +logout = None + +def get_level(x, default=ERROR): + try: + return int(x) + except ValueError: + pass + try: + return loglevels[x] + except KeyError: + return default + +def err(*msg, **kwargs): + global collector, logfile + if 'nologerror' not in kwargs: + if collector is None: + try: + collector = FilteredFileLogObserver(StringIO(), ERROR) + collector.start() + except: pass + if logfile is None: + try: + logfile = FilteredFileLogObserver(open(Config.get('logfile','audiomangler-%d.log' % os.getpid()), 'wb'), get_level(Config['loglevel'])) + logfile.start() + except: pass + kwargs['loglevel'] = ERROR + if msg and isinstance(msg[0],(failure.Failure,Exception)): + log.err(*msg, **kwargs) + else: + log.msg(*msg, **kwargs) + +def msg(*msg, **kwargs): + global logfile + kwargs.setdefault('loglevel',DEBUG) + if kwargs['loglevel'] == ERROR: + err(*msg, **kwargs) + else: + if Config['logfile'] and logfile is None and kwargs['loglevel'] <= get_level(Config['loglevel']): + try: + logfile = FilteredFileLogObserver(open(Config['logfile'], 'wb'), Config['loglevel']) + logfile.start() + except: pass + log.msg(*msg, **kwargs) + +def fatal(*msg, **kwargs): + err(*msg, **kwargs) + sys.exit() + +def cleanup(): + if logout: + sys.stdout.flush() + logout.stop() + if collector: + collector.output.flush() + collector.stop() + text = collector.output.getvalue() + if text: + print "The following errors occurred:" + print text + print "The above errors may also have been reported during processing." + if logfile: + logfile.output.flush() + logfile.stop() + print "Errors are also recorded in the logfile '%s'." % os.path.abspath(logfile.output.name) + sys.stdout.flush() + +try: + logout = FilteredConsoleLogObserver(Config['consolelevel']) + logout.start() + if log.defaultObserver: + log.defaultObserver.stop() + log.defaultObserver = None +except: + pass + +atexit.register(cleanup) + +__all__ = ['err','msg','fatal','ERROR','WARNING','INFO','DEBUG'] \ No newline at end of file diff --git a/audiomangler/scanner.py b/audiomangler/scanner.py index 83aa1d0..408056e 100644 --- a/audiomangler/scanner.py +++ b/audiomangler/scanner.py @@ -161,7 +161,7 @@ def scan(items, groupby = None, sortby = None, trackid = None): for t in tracks: t.sortkey = t.meta.evaluate(sortby) albums.setdefault(t.meta.evaluate(groupby),[]).append(t) - dirs.setdefault(t.meta['dir'],[]).append(t) + dirs.setdefault(os.path.split(t.filename)[0],[]).append(t) t.tid = t.meta.evaluate(trackid) if t.tid in trackids: print "trackid collision" diff --git a/audiomangler/task.py b/audiomangler/task.py index 52cd25a..93da224 100644 --- a/audiomangler/task.py +++ b/audiomangler/task.py @@ -8,27 +8,27 @@ ########################################################################### import os import sys -from subprocess import Popen, CalledProcessError -from threading import Thread, RLock +import atexit from types import FunctionType, GeneratorType -threadpool = None +from twisted.internet import threads, defer +from twisted.python import failure + +if 'twisted.internet.reactor' not in sys.modules: + for reactor in ('kqreactor','epollreactor','pollreactor','selectreactor'): + try: + r = __import__('twisted.internet.' + reactor, fromlist=[reactor]) + r.install() + break + except ImportError: pass -def synchronized(*locks): - def decorator(func): - def proxy(self, *args, **kw): - for lock in locks: - lock = getattr(self,lock,None) - if lock is not None: - lock.acquire() - try: - return func(self, *args, **kw) - finally: - for lock in locks: - lock = getattr(self,lock,None) - if lock is not None: - lock.release() - return proxy - return decorator +from twisted.internet import reactor + +def cleanup(): + if reactor.running: reactor.stop() + +atexit.register(cleanup) + +threadpool = None class Clear(object): def __new__(cls,*args,**kw): @@ -45,54 +45,26 @@ class BaseTask(object): def run(self): self.started = True if self.background: - self.runbg() - return self + return self.runbg() else: - self.runfg() + return self.runfg() class FuncTask(BaseTask): def __init__(self, target=None, args=(), kwargs=(), background=True): - self._lock = RLock() BaseTask.__init__(self, target, args, kwargs, background) - run = synchronized('_lock')(BaseTask.run) - def runfg(self): - self.target(*self.args, **self.kwargs) - - @synchronized('_lock') - def runbg(self): - self.thread = Thread(target=self.runsaveexc) - self.thread.start() - - def runsaveexc(self): try: - self.runfg() - except Exception: - self.exc_info = sys.exc_info() + self.target(*self.args, **self.kwargs) + self.deferred = defer.succeed(self) + return self.deferred + except: + self.deferred = defer.failure(failure.Failure()) + return self.deferred - @synchronized('_lock') - def wait(self): - if not self.started: - return False - elif not hasattr(self,'thread'): - return False - self.thread.join() - if hasattr(self, 'exc_info'): - sys.excepthook(*exc_info) - return True - - @synchronized('_lock') - def poll(self): - if not self.started: - return False - elif not hasattr(self,'thread'): - return False - elif self.thread.isAlive(): - return False - elif hasattr(self, 'exc_info'): - sys.excepthook(*(self.exc_info)) - return True + def runbg(self): + self.deferred = threads.deferToThread(self.target, *self.args, **self.kwargs) + return self.deferred class CLITask(BaseTask): def __init__(self, target=None, args=(), stdin=None, stdout=None, stderr=None, kwargs=(), background=True): @@ -165,4 +137,4 @@ class TaskSet(FuncTask): self.clear() self.clear() -__all__ = ['FuncTask','CLITask','TaskSet'] +__all__ = ['FuncTask','CLITask','TaskSet','reactor'] diff --git a/audiomangler/util.py b/audiomangler/util.py index ea7833f..3309b47 100644 --- a/audiomangler/util.py +++ b/audiomangler/util.py @@ -7,6 +7,7 @@ # ########################################################################### import os, stat +from audiomangler import Config, msg, err, fatal, WARNING, ERROR def copy(src,dst): fsrc = None @@ -39,4 +40,37 @@ def move(src,dst): copy(src,dst) os.unlink(src) -__all__ = ['copy','move'] \ No newline at end of file +def test_splits(dir_list,transcode=False): + from audiomangler import get_codec + if transcode and Config['type']: + targetcodec = Config['type'] + if ',' in targetcodec: + allowedcodecs = targetcodec.split(',') + targetcodec = allowedcodecs[0] + allowedcodecs = frozenset(allowedcodecs) + else: + allowedcodecs = frozenset((targetcodec,)) + targetcodec = get_codec(targetcodec) + postadd = lambda type_: {} if type_ in allowedcodecs else {'type':targetcodec.type_,'ext':targetcodec.ext} + else: + postadd = lambda type_: {} + for (dir_, files) in dir_list.items(): + dstdirs = set() + for file_ in files: + src = file_.filename + dst = fsencode(file_.format(postadd=postadd(file_.type_))) + dstdir = os.path.split(dst)[0] + dstdirs.add(dstdir) + if len(dstdirs) > 1: + onsplit = Config['onsplit'] + if onsplit == 'abort': + fatal(consoleformat=u"tracks in %(dir_p)s would be placed in different target directories, aborting\nset onsplit to 'warn' or 'ignore' to proceed anyway", + format="split: %(dir_)r", dir_=dir_, dir_p=fsdecode(dir_),nologerror=1) + +def fsencode(string): + return string.encode(Config['fs_encoding'],Config.get('fs_encoding_err','underscorereplace')) + +def fsdecode(string): + return string.decode(Config['fs_encoding'],Config.get('fs_encoding_err','replace')) + +__all__ = ['copy','move','fsencode','fsdecode','test_splits'] \ No newline at end of file -- 2.11.4.GIT