build/checksums.py

   1 #!/usr/bin/python
   2 # ***** BEGIN LICENSE BLOCK *****
   3 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   4 #
   5 # The contents of this file are subject to the Mozilla Public License Version
   6 # 1.1 (the "License"); you may not use this file except in compliance with
   7 # the License. You may obtain a copy of the License at
   8 # http://www.mozilla.org/MPL/
   9 #
  10 # Software distributed under the License is distributed on an "AS IS" basis,
  11 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  12 # for the specific language governing rights and limitations under the
  13 # License.
  14 #
  15 # The Original Code is a checksum generation utility.
  16 #
  17 # The Initial Developer of the Original Code is
  18 # Mozilla Foundation.
  19 # Portions created by the Initial Developer are Copyright (C) 2010
  20 # the Initial Developer. All Rights Reserved.
  21 #
  22 # Contributor(s):
  23 #    John Ford <jhford@mozilla.com>
  24 #
  25 # Alternatively, the contents of this file may be used under the terms of
  26 # either the GNU General Public License Version 2 or later (the "GPL"), or
  27 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  28 # in which case the provisions of the GPL or the LGPL are applicable instead
  29 # of those above. If you wish to allow use of your version of this file only
  30 # under the terms of either the GPL or the LGPL, and not to allow others to
  31 # use your version of this file under the terms of the MPL, indicate your
  32 # decision by deleting the provisions above and replace them with the notice
  33 # and other provisions required by the GPL or the LGPL. If you do not delete
  34 # the provisions above, a recipient may use your version of this file under
  35 # the terms of any one of the MPL, the GPL or the LGPL.
  36 #
  37 # ***** END LICENSE BLOCK *****
  38
  39 from optparse import OptionParser
  40 import logging
  41 import os
  42 try:
  43     import hashlib
  44 except:
  45     hashlib = None
  46
  47 def digest_file(filename, digest, chunk_size=1024):
  48     '''Produce a checksum for the file specified by 'filename'.  'filename'
  49     is a string path to a file that is opened and read in this function.  The
  50     checksum algorithm is specified by 'digest' and is a valid OpenSSL
  51     algorithm.  If the digest used is not valid or Python's hashlib doesn't
  52     work, the None object will be returned instead.  The size of blocks
  53     that this function will read from the file object it opens based on
  54     'filename' can be specified by 'chunk_size', which defaults to 1K'''
  55     assert not os.path.isdir(filename), 'this function only works with files'
  56     logger = logging.getLogger('checksums.py')
  57     if hashlib is not None:
  58         logger.debug('Creating new %s object' % digest)
  59         h = hashlib.new(digest)
  60         f = open(filename)
  61         while True:
  62             data = f.read(chunk_size)
  63             if not data:
  64                 logger.debug('Finished reading in file')
  65                 break
  66             h.update(data)
  67         f.close()
  68         hash = h.hexdigest()
  69         logger.debug('Hash for %s is %s' % (filename, hash))
  70         return hash
  71     else:
  72         # In this case we could subprocess.Popen and .communicate with
  73         # sha1sum or md5sum
  74         logger.warn('The python module for hashlib is missing!')
  75         return None
  76
  77
  78 def process_files(files, output_filename, digest, strip):
  79     '''This function takes a list of file names, 'files'.  It will then
  80     compute the checksum for each of the files by opening the files.
  81     Once each file is read and its checksum is computed, this function
  82     will write the information to the file specified by 'output_filename'.
  83     The path written in the output file will have anything specified by 'strip'
  84     removed from the path.  The output file is closed before returning nothing
  85     The algorithm to compute checksums with can be specified by 'digest'
  86     and needs to be a valid OpenSSL algorithm.
  87
  88     The output file is written in the format:
  89         <hash> <algorithm> <filesize> <filepath>
  90     Example:
  91         d1fa09a<snip>e4220 sha1 14250744 firefox-4.0b6pre.en-US.mac64.dmg
  92     '''
  93
  94     logger = logging.getLogger('checksums.py')
  95     if os.path.exists(output_filename):
  96         logger.debug('Overwriting existing checksums file "%s"' %
  97                      output_filename)
  98     else:
  99         logger.debug('Creating a new checksums file "%s"' % output_filename)
 100     output = open(output_filename, 'w+')
 101     for file in files:
 102         if os.path.isdir(file):
 103             logger.warn('%s is a directory, skipping' % file)
 104         else:
 105             hash = digest_file(file, digest)
 106             if hash is None:
 107                 logger.warn('Unable to generate a hash for %s. ' +
 108                             'Using NOHASH as fallback' % file)
 109                 hash = 'NOHASH'
 110             if file.startswith(strip):
 111                 short_file = file[len(strip):]
 112                 short_file = short_file.lstrip('/')
 113             else:
 114                 short_file = file
 115             print >>output, '%s %s %s %s' % (hash, digest,
 116                                              os.path.getsize(file),
 117                                              short_file)
 118     output.close()
 119
 120 def setup_logging(level=logging.DEBUG):
 121     '''This function sets up the logging module using a speficiable logging
 122     module logging level.  The default log level is DEBUG.
 123
 124     The output is in the format:
 125         <level> - <message>
 126     Example:
 127         DEBUG - Finished reading in file
 128 '''
 129
 130     logger = logging.getLogger('checksums.py')
 131     logger.setLevel(logging.DEBUG)
 132     handler = logging.StreamHandler()
 133     handler.setLevel(level)
 134     formatter = logging.Formatter("%(levelname)s - %(message)s")
 135     handler.setFormatter(formatter)
 136     logger.addHandler(handler)
 137
 138 def main():
 139     '''This is a main function that parses arguments, sets up logging
 140     and generates a checksum file'''
 141     # Parse command line arguments
 142     parser = OptionParser()
 143     parser.add_option('-d', '--digest', help='checksum algorithm to use',
 144                       action='store', dest='digest', default='sha1')
 145     parser.add_option('-o', '--output', help='output file to use',
 146                       action='store', dest='outfile', default='checksums')
 147     parser.add_option('-v', '--verbose',
 148                       help='Be noisy (takes precedence over quiet)',
 149                       action='store_true', dest='verbose', default=False)
 150     parser.add_option('-q', '--quiet', help='Be quiet', action='store_true',
 151                       dest='quiet', default=False)
 152     parser.add_option('-s', '--strip',
 153                       help='strip this path from the filenames',
 154                       dest='strip', default=os.getcwd())
 155     options, args = parser.parse_args()
 156
 157     #Figure out which logging level to use
 158     if options.verbose:
 159         loglevel = logging.DEBUG
 160     elif options.quiet:
 161         loglevel = logging.ERROR
 162     else:
 163         loglevel = logging.INFO
 164
 165     #Set up logging
 166     setup_logging(loglevel)
 167     logger = logging.getLogger('checksums.py')
 168
 169     # Validate the digest type to use
 170     try:
 171         hashlib.new(options.digest)
 172     except ValueError, ve:
 173         logger.error('Could not create a "%s" hash object (%s)' %
 174                      (options.digest, ve.args[0]))
 175         exit(1)
 176
 177     # Validate the files to checksum
 178     files = []
 179     for i in args:
 180         if os.path.exists(i):
 181             files.append(i)
 182         else:
 183             logger.info('File "%s" was not found on the filesystem' % i)
 184     process_files(files, options.outfile, options.digest, options.strip)
 185
 186 if __name__ == '__main__':
 187     main()