From 8a946e96b81ac759209e4fa206974cd546bd8e59 Mon Sep 17 00:00:00 2001 From: "Adam J. Gamble" Date: Sat, 24 Nov 2012 15:25:03 +0000 Subject: [PATCH] Major; refactored to use CsvCommand for initializing collate objects. * squashed commits from branch csvbase-command (closed). --- colly/commands/__init__.py | 96 ++++++++++++++++++++++++++++++++++++++++++- colly/commands/diff.py | 57 +++++++++++++++++++++++++ colly/commands/format.py | 90 ++++++++++++---------------------------- colly/{collate.py => core.py} | 3 ++ 4 files changed, 182 insertions(+), 64 deletions(-) rewrite colly/commands/format.py (75%) rename colly/{collate.py => core.py} (98%) diff --git a/colly/commands/__init__.py b/colly/commands/__init__.py index 0352b34..a1431f2 100644 --- a/colly/commands/__init__.py +++ b/colly/commands/__init__.py @@ -2,13 +2,18 @@ import sys from pkgutil import walk_packages import logging import optparse + from colly.optbase import parser +from colly.core import Collate __all__ = ['command_dict', 'Command', 'load_command', 'load_all_commands', 'command_names'] command_dict = {} +''' Base command +''' + class Command(object): name = None usage = None @@ -28,7 +33,96 @@ class Command(object): def main(self, initial_options, args): options, args = self.parser.parse_args(args) - self.run(options, args) + self.run(options, args) + + +''' CSV base command + -- + @ Parses common options to initialize colly's core(.py) object +''' + +class CsvCommand(Command): + usage = "%prog FILE(S) [OPTIONS]" + objects = [] + + def __init__(self): + super(CsvCommand, self).__init__() + + self.parser.add_option('-H', '--headings', + dest='headings', + action='callback', + callback=split_callback, + default=[], + type='str', + help='Comma separated headings for CSV columns') + + self.parser.add_option('-i', '--index', + dest='index', + action='callback', + callback=split_callback, + default=[], + type='str', + help='Define index column on CSV, shortcut to using -H pk,etc.') + + ''' + TODO: + self.parser.add_option('-z', '--lazy', + dest='lazy', + action='store_true', + help='Will make assumptions rather than throw errors') + + self.parser.add_option('-i', '--index', + dest='pk', + action='store', + help='Column to set as the index (pk)') + + self.parser.add_option('-f', '--format', + dest='format', + action='store', + help='Output format [default: JSON]') + + self.parser.add_option('-c', '--compress', + dest='compress', + action='store_true', + help='Column to set as the index (pk)') + ''' + + def main(self, initial_options, args): #:! overwrites Command.main + options, args = self.parser.parse_args(args) + + if options.headings and options.index: + self.parser.error('Options HEADINGS and INDEX are mutually exclusive') + + n = 0 + for csv_file in args: + kwargs = {} + + if options.headings: + try: + kwargs['headings'] = options.headings[n] + except IndexError: + break + elif options.index: + try: + kwargs['headings'] = [] # leave out index for now + except IndexError: + break + else: + pass + + self.objects.append(Collate(csv_file, **kwargs)) + n += 1 + + self.run(options, args) + +# Utils + +def split_callback(option, opt, value, parser): + ''' Callback, extra option parsing + ''' + value_list = map(lambda v: v.split(','), value.split(':')) + setattr(parser.values, option.dest, value_list) + def load_command(name): full_name = 'colly.commands.%s' % name diff --git a/colly/commands/diff.py b/colly/commands/diff.py index e69de29..8440292 100644 --- a/colly/commands/diff.py +++ b/colly/commands/diff.py @@ -0,0 +1,57 @@ +import sys +import logging + +from colly.commands import CsvCommand + +class DiffCommand(CsvCommand): + name = 'diff' + summary = "Compare two CSV files by index column (pk)" + + def __init__(self): + super(DiffCommand, self).__init__() + + self.parser.add_option('-a', '--added', + dest="method", # i.e. the set method to call + action="store_const", + const="added", + help='Show rows in CSV 2 not in CSV 1.') + + self.parser.add_option('-e', '--erased', + dest="method", + action="store_const", + const="erased", + help='Show rows in CSV 1 not in CSV 2.') + + self.parser.add_option('-c', '--clean', + dest="method", + action="store_const", + const="clean", + help='Show rows in CSV 1 and in CSV 2.') + + self.parser.add_option('-A', '--all', + dest="method", + action="store_const", + const="all", + help='Show all rows in CSV 1 and CSV 2.') + + def run(self, options, args): + A = self.objects[0] + B = self.objects[1] + + if not options.method: + self.parser.error('Use one of the options [aAce] make a comparison.') + + filtered = { + 'added': B.column.difference(A.column), + 'erased': A.column.difference(B.column), + 'clean': A.column.intersection(B.column), + 'all' : A.column.union(B.column) + }[options.method] + + for row in filtered: + try: + print A.get_row(row) + except KeyError: + print B.get_row(row) + +DiffCommand() diff --git a/colly/commands/format.py b/colly/commands/format.py dissimilarity index 75% index 0cd2cfc..02343d3 100644 --- a/colly/commands/format.py +++ b/colly/commands/format.py @@ -1,63 +1,27 @@ -import sys -import logging - -from colly.commands import Command -from colly.collate import Collate - -def headings_callback(option, opt, value, parser): - setattr(parser.values, option.dest, value.split(',')) - logging.warning(value.split(',')) - -class FormatCommand(Command): - name = 'format' - usage = "%prog FILE [OPTIONS]" #: overwrites baseparser - summary = "Turn collated CSV file into JSON, or other format" - - def __init__(self): - super(FormatCommand, self).__init__() - - self.parser.add_option('-H', '--headings', - dest='headings', - action='callback', - callback=headings_callback, - default=[], - type='str', - help='Comma separated headings for CSV columns') - - ''' - TODO: - self.parser.add_option('-z', '--lazy', - dest='lazy', - action='store_true', - help='Will make assumptions rather than throw errors') - - self.parser.add_option('-i', '--index', - dest='pk', - action='store', - help='Column to set as the index (pk)') - - self.parser.add_option('-f', '--format', - dest='format', - action='store', - help='Output format [default: JSON]') - - self.parser.add_option('-c', '--compress', - dest='compress', - action='store_true', - help='Column to set as the index (pk)') - ''' - - def run(self, options, args): - try: - csv_file = args[0] - except IndexError: - exit('CSV file required as first argument') #: FIX: raise properly. - - kwargs = { - 'headings': options.headings - } - - ix = Collate(csv_file, **kwargs) - print ix.as_json() - -FormatCommand() +import sys +import logging + +from colly.commands import CsvCommand + +class FormatCommand(CsvCommand): + name = 'format' + usage = "%prog FILE [OPTIONS]" #: overwrites baseparser + summary = "Turn collated CSV file into JSON, or other format" + + def __init__(self): + super(FormatCommand, self).__init__() + + ''' + self.parser.add_option('-f', '--format', + dest='format', + action='callback', + callback=headings_callback, + default='json', + type='str', + help='Convert to format (-f) from CSV.') + ''' + + def run(self, options, args): + print self.objects[0].as_json() + +FormatCommand() diff --git a/colly/collate.py b/colly/core.py similarity index 98% rename from colly/collate.py rename to colly/core.py index 167f957..6424f49 100644 --- a/colly/collate.py +++ b/colly/core.py @@ -83,6 +83,9 @@ class Collate(object): ''' pass + def get_row(self, pk): + return self._rows[pk] + ''' Rehashing/ formatting ''' def as_json(self): -- 2.11.4.GIT