assurance/main.py

   1 #! /usr/bin/env python
   2
   3 # Directory integrity scanner.
   4
   5 from stat import *
   6 import os
   7 import sys
   8 from os.path import join
   9
  10 from cPickle import dump, load
  11 import gzip
  12
  13 import hashing
  14
  15 def walk(top):
  16     """Root of directory generator"""
  17     topstat = os.lstat(top)
  18     for x in walker(top, '.', topstat):
  19         yield x
  20
  21 def walker(path, name, dirstat):
  22     """Directory tree generator.
  23
  24     At one point, this started as a copy of os.walk from Python's
  25     library.  Even the arguments are different now.
  26     """
  27
  28     try:
  29         names = os.listdir(path)
  30     except OSError:
  31         sys.stderr.write("Warning, can't read dir: %s\n" % path)
  32         return
  33
  34     # The verification algorithm requires the names to be sorted.
  35     names.sort()
  36
  37     # Stat each name found, and put the result in one of two lists.
  38     dirs, nondirs = [], []
  39     for onename in names:
  40         if path == '.' and (onename == "0sure.dat.gz" or
  41                 onename == "0sure.bak.gz" or
  42                 onename == "0sure.0.gz"):
  43             continue
  44         st = os.lstat(join(path, onename))
  45         if S_ISDIR(st.st_mode):
  46             dirs.append((onename, st))
  47         else:
  48             nondirs.append((onename, st))
  49
  50     # Indicate "entering" the directory.
  51     yield 'd', name, convert_stat(dirstat)
  52
  53     # Then recursively walk into all of the subdirectories.
  54     for (onename, st) in dirs:
  55         subpath = join(path, onename)
  56         if st.st_dev == dirstat.st_dev:
  57             for x in walker(subpath, onename, st):
  58                 yield x
  59
  60     # Then yield each entry that is not a subdirectory.
  61     for (onename, st) in nondirs:
  62         yield '-', onename, convert_stat(st)
  63
  64     # Last, yield the leaving.
  65     yield ('u',)
  66
  67 # Convert the passed stat info into an association of the information
  68 # itself.  Does not do anything that requires reading the file (such
  69 # as readlink or md5).
  70 def convert_stat(st):
  71     if S_ISDIR(st.st_mode):
  72         return { 'kind': 'dir',
  73                  'uid': st.st_uid,
  74                  'gid': st.st_gid,
  75                  'perm': S_IMODE(st.st_mode) }
  76
  77     elif S_ISREG(st.st_mode):
  78         return { 'kind': 'file',
  79                  'uid': st.st_uid,
  80                  'gid': st.st_gid,
  81                  'mtime': st.st_mtime,
  82                  'ctime': st.st_ctime,
  83                  'ino': st.st_ino,
  84                  'perm': S_IMODE(st.st_mode) }
  85
  86     elif S_ISLNK(st.st_mode):
  87         return { 'kind': 'lnk' }
  88
  89     elif S_ISSOCK:
  90         return { 'kind': 'sock',
  91                  'uid': st.st_uid,
  92                  'gid': st.st_gid,
  93                  'perm': S_IMODE(st.st_mode) }
  94
  95     elif S_ISFIFO:
  96         return { 'kind': 'fifo',
  97                  'uid': st.st_uid,
  98                  'gid': st.st_gid,
  99                  'perm': S_IMODE(st.st_mode) }
 100
 101     elif S_ISBLK:
 102         return { 'kind': 'blk',
 103                  'uid': st.st_uid,
 104                  'gid': st.st_gid,
 105                  'devmaj': os.major(st.st_rdev),
 106                  'devmin': os.minor(st.st_rdev),
 107                  'perm': S_IMODE(st.st_mode) }
 108
 109     elif S_ISCHR:
 110         return { 'kind': 'chr',
 111                  'uid': st.st_uid,
 112                  'gid': st.st_gid,
 113                  'devmaj': os.major(st.st_rdev),
 114                  'devmin': os.minor(st.st_rdev),
 115                  'perm': S_IMODE(st.st_mode) }
 116
 117     else:
 118         raise "Unknown file kind"
 119
 120 def empty_tree():
 121     """Make an empty tree.  No meaningful attributes for the root
 122     directory"""
 123     yield 'd', '.', {}
 124     yield 'u',
 125     return
 126
 127 def empty_generator():
 128     return
 129     yield ()
 130
 131 class comparer:
 132     """Class for comparing two directory iterations.  Keeps track of
 133     state, and allows child classes to define handlers for the various
 134     types of differences found."""
 135
 136     def __init__(self, left, right):
 137         self.__left = left
 138         self.__right = right
 139
 140     # Default handlers for the 6 possible changes (or not changes)
 141     # that can happen in a directory.  The adds and deletes take an
 142     # additional argument that will be set to true if this added or
 143     # remoted entity is contained in an entirely new directory.  Some
 144     # handlers may want to avoid printing verbose messages for the
 145     # contents of added or deleted directories, and can use this
 146     # value.
 147     def handle_same_dir(self, path, a, b):
 148         #print "same_dir(%s, %s, %s)" % (path, a, b)
 149         return empty_generator()
 150     def handle_delete_dir(self, path, a, recursing):
 151         #print "delete_dir(%s, %s, %s)" % (path, a, recursing)
 152         return empty_generator()
 153     def handle_add_dir(self, path, a, recursing):
 154         #print "add_dir(%s, %s, %s)" % (path, a, recursing)
 155         return empty_generator()
 156     def handle_same_nondir(self, path, a, b):
 157         #print "same_nondir(%s, %s, %s)" % (path, a, b)
 158         return empty_generator()
 159     def handle_delete_nondir(self, path, a, recursing):
 160         #print "delete_nondir(%s, %s, %s)" % (path, a, recursing)
 161         return empty_generator()
 162     def handle_add_nondir(self, path, a, recursing):
 163         #print "add_nondir(%s, %s, %s)" % (path, a, recursing)
 164         return empty_generator()
 165     def handle_leave(self, path, recursing):
 166         return empty_generator()
 167
 168     def run(self):
 169         a = self.__left.next()
 170         if a[0] != 'd':
 171             raise "Scan doesn't start with a directory"
 172         b = self.__right.next()
 173         if b[0] != 'd':
 174             raise "Tree walk doesn't start with a directory"
 175         for x in self.handle_same_dir(".", a, b):
 176             yield x
 177         for x in self.__run(b[1], 1):
 178             yield x
 179
 180     def __run(self, path, depth):
 181         """Iterate both pairs of directories equally
 182
 183         Processes the contents of a single directory, recursively
 184         calling itself to handle child directories.  Returns with both
 185         iterators advanced past the 'u' node that ends the dir."""
 186         # print "run(%d): '%s'" % (depth, path)
 187         a = self.__left.next()
 188         b = self.__right.next()
 189
 190         while True:
 191             # print "Comparing (%d) %s and %s" % (depth, a, b)
 192             if a[0] == 'u' and b[0] == 'u':
 193                 # Both are leaving the directory.
 194                 # print "leave(%d): '%s'" % (depth, path)
 195                 for x in self.handle_leave(path, False):
 196                     yield x
 197                 return
 198
 199             elif a[0] == 'd' and b[0] == 'd':
 200                 # Both looking at a directory entry.
 201
 202                 if a[1] == b[1]:
 203                     # if the name is the same, walk the tree.
 204                     for x in self.handle_same_dir(path, a, b):
 205                         yield x
 206                     for x in self.__run(os.path.join(path, a[1]), depth + 1):
 207                         yield x
 208                     a = self.__left.next()
 209                     b = self.__right.next()
 210                     continue
 211
 212                 elif a[1] < b[1]:
 213                     # A directory has been deleted.
 214                     for x in self.handle_delete_dir(path, a, False):
 215                         yield x
 216                     for x in self.delete_whole_dir(self.__left,
 217                             os.path.join(path, a[1])):
 218                         yield x
 219                     a = self.__left.next()
 220                     continue
 221
 222                 else:
 223                     # A directory has been added.
 224                     for x in self.handle_add_dir(path, b, False):
 225                         yield x
 226
 227                     for x in self.add_whole_dir(self.__right,
 228                             os.path.join(path, b[1])):
 229                         yield x
 230                     b = self.__right.next()
 231                     continue
 232
 233             elif a[0] == '-' and b[0] == '-':
 234                 # Both are looking at a non-dir.
 235
 236                 if a[1] == b[1]:
 237                     # Same name as well.
 238                     for x in self.handle_same_nondir(path, a, b):
 239                         yield x
 240                     a = self.__left.next()
 241                     b = self.__right.next()
 242                     continue
 243
 244                 elif a[1] < b[1]:
 245                     # Deleted non-dir.
 246                     for x in self.handle_delete_nondir(path, a, False):
 247                         yield x
 248                     a = self.__left.next()
 249                     continue
 250
 251                 else:
 252                     # Added non-dir.
 253                     for x in self.handle_add_nondir(path, b, False):
 254                         yield x
 255                     b = self.__right.next()
 256                     continue
 257
 258             elif a[0] == '-' and b[0] == 'u':
 259                 for x in self.handle_delete_nondir(path, a, False):
 260                     yield x
 261                 a = self.__left.next()
 262                 continue
 263
 264             elif a[0] == 'u' and b[0] == '-':
 265                 for x in self.handle_add_nondir(path, b, False):
 266                     yield x
 267                 b = self.__right.next()
 268                 continue
 269
 270             elif a[0] == 'd' and (b[0] == '-' or b[0] == 'u'):
 271                 for x in self.handle_delete_dir(path, a, False):
 272                     yield x
 273                 for x in self.delete_whole_dir(self.__left,
 274                         os.path.join(path, a[1])):
 275                     yield x
 276                 a = self.__left.next()
 277                 continue
 278
 279             elif (a[0] == '-' or a[0] == 'u') and b[0] == 'd':
 280                 for x in self.handle_add_dir(path, b, False):
 281                     yield x
 282                 for x in self.add_whole_dir(self.__right,
 283                     os.path.join(path, b[1])):
 284                     yield x
 285                 b = self.__right.next()
 286                 continue
 287
 288             else:
 289                 print "Unhandled case: '%s' and '%s'" % (a[0], b[0])
 290                 sys.exit(2)
 291
 292     def add_whole_dir(self, iter, path):
 293         "Consume entries until this directory has been added"
 294         # print "add_whole_dir: %s" % path
 295         while True:
 296             a = iter.next()
 297             if a[0] == 'u':
 298                 for x in self.handle_leave(path, True):
 299                     yield x
 300                 return
 301             elif a[0] == 'd':
 302                 for x in self.handle_add_dir(path, a, True):
 303                     yield x
 304                 for x in self.add_whole_dir(iter, os.path.join(path, a[1])):
 305                     yield x
 306             else:
 307                 for x in self.handle_add_nondir(path, a, True):
 308                     yield x
 309
 310     def delete_whole_dir(self, iter, path):
 311         "Consume entries until this directory has been deleted"
 312         # print "delete_whole_dir: %s" % path
 313         while True:
 314             a = iter.next()
 315             if a[0] == 'u':
 316                 for x in self.handle_leave(path, True):
 317                     yield x
 318                 return
 319             elif a[0] == 'd':
 320                 for x in self.handle_delete_dir(path, a, True):
 321                     yield x
 322                 for x in self.delete_whole_dir(iter, os.path.join(path, a[1])):
 323                     yield x
 324             else:
 325                 for x in self.handle_delete_nondir(path, a, True):
 326                     yield x
 327
 328 __must_match = {
 329         'dir': ['uid', 'gid', 'perm'],
 330         'file': ['uid', 'gid', 'mtime', 'perm', 'md5'],
 331         'lnk': ['targ'],
 332         'sock': ['uid', 'gid', 'perm'],
 333         'fifo': ['uid', 'gid', 'perm'],
 334         'blk': ['uid', 'gid', 'perm', 'devmaj', 'devmin'],
 335         'chr': ['uid', 'gid', 'perm', 'devmaj', 'devmin'],
 336         }
 337 def compare_entries(path, a, b):
 338     if a['kind'] != b['kind']:
 339         yield "- %-4s %s" % (a['kind'], path)
 340         yield "+ %-4s %s" % (b['kind'], path)
 341         return
 342     misses = []
 343     for item in __must_match[a['kind']]:
 344         if not (a.has_key(item) and b.has_key(item)):
 345             misses.append(item)
 346         elif a[item] != b[item]:
 347             misses.append(item)
 348     if misses:
 349         yield "[%s] %s" % (",".join(misses), path)
 350     if 'targ' in misses:
 351         if a.has_key('targ'):
 352             yield "  old targ: %s" % a['targ']
 353         if b.has_key('targ'):
 354             yield "  new targ: %s" % b['targ']
 355     return
 356
 357 class check_comparer(comparer):
 358     """Comparer for comparing either two trees, or a tree and a
 359     filesystem.  'right' should be the newer tree.
 360     Yields strings giving the tree differences.
 361     """
 362     def handle_same_dir(self, path, a, b):
 363         return compare_entries(os.path.join(path, a[1]), a[2], b[2])
 364
 365     def handle_delete_dir(self, path, a, recursing):
 366         if recursing:
 367             return
 368         else:
 369             yield "- dir  %s" % (os.path.join(path, a[1]))
 370     def handle_add_dir(self, path, a, recursing):
 371         if recursing:
 372             return
 373         else:
 374             yield "+ dir  %s" % (os.path.join(path, a[1]))
 375     def handle_same_nondir(self, path, a, b):
 376         return compare_entries(os.path.join(path, a[1]), a[2], b[2])
 377
 378     def handle_delete_nondir(self, path, a, recursing):
 379         if recursing:
 380             return
 381         else:
 382             yield "-      %s" % (os.path.join(path, a[1]))
 383     def handle_add_nondir(self, path, a, recursing):
 384         if recursing:
 385             return
 386         else:
 387             yield "+      %s" % (os.path.join(path, a[1]))
 388
 389 def update_link(assoc, path, name):
 390     if assoc['kind'] == 'lnk':
 391         assoc['targ'] = os.readlink(os.path.join(path, name))
 392
 393 def same_inode(a, b):
 394     """Do these two nodes reference what appears to be the same,
 395     unmodified inode."""
 396     return (a['kind'] == b['kind'] and
 397             a['ino'] == b['ino'] and
 398             a['ctime'] == b['ctime'])
 399
 400 class update_comparer(comparer):
 401     """Yields a tree equivalent to the right tree, which should be
 402     coming from a live filesystem.  Fills in symlink destinations and
 403     file md5sums (if possible)."""
 404
 405     def handle_same_dir(self, path, a, b):
 406         yield b
 407         return
 408
 409     def handle_add_dir(self, path, a, recursing):
 410         yield a
 411         return
 412
 413     def handle_same_nondir(self, path, a, b):
 414         update_link(b[2], path, b[1])
 415         if b[2]['kind'] == 'file':
 416             if same_inode(a[2], b[2]):
 417                 b[2]['md5'] = a[2]['md5']
 418             else:
 419                 b[2]['md5'] = hashing.hashof(os.path.join(path, b[1]))
 420         yield b
 421         return
 422
 423     def handle_add_nondir(self, path, a, recursing):
 424         update_link(a[2], path, a[1])
 425         if a[2]['kind'] == 'file':
 426             a[2]['md5'] = hashing.hashof(os.path.join(path, a[1]))
 427         yield a
 428         return
 429
 430     def handle_leave(self, path, recursing):
 431         yield 'u',
 432         return
 433
 434 version = 'Asure scan version 1.0'
 435
 436 def reader(path):
 437     """Iterate over a previously written dump"""
 438     fd = gzip.open(path, 'rb')
 439     vers = load(fd)
 440     if version != vers:
 441         raise "incompatible version of asure file"
 442     try:
 443         while True:
 444             yield load(fd)
 445     except EOFError:
 446         return
 447
 448 def writer(path, iter):
 449     """Write the given item (probably assembled iterator)"""
 450     fd = gzip.open(path, 'wb')
 451     dump(version, fd, -1)
 452     for item in iter:
 453         # print item
 454         dump(item, fd, -1)
 455     fd.close
 456
 457 def rename_cycle():
 458     """Cycle through the names"""
 459     try:
 460         os.rename('0sure.dat.gz', '0sure.bak.gz')
 461     except OSError:
 462         pass
 463     os.rename('0sure.0.gz', '0sure.dat.gz')
 464
 465 def fresh_scan():
 466     """Perform a fresh scan of the filesystem"""
 467     tree = update_comparer(empty_tree(), walk('.'))
 468     writer('0sure.0.gz', tree.run())
 469     rename_cycle()
 470
 471 def check_scan():
 472     """Perform a scan of the filesystem, and compare it with the scan
 473     file.  reports differences."""
 474     prior = reader('0sure.dat.gz')
 475     cur = update_comparer(empty_tree(), walk('.')).run()
 476     # compare_trees(prior, cur)
 477     for x in check_comparer(prior, cur).run():
 478         print x
 479
 480 def update():
 481     """Scan filesystem, but also read the previous scan to cache md5
 482     hashes of files that haven't had any inode changes"""
 483     prior = reader('0sure.dat.gz')
 484     cur = update_comparer(prior, walk('.')).run()
 485     writer('0sure.0.gz', cur)
 486     rename_cycle()
 487
 488 def signoff():
 489     """Compare the previous scan with the current."""
 490     prior = reader('0sure.bak.gz')
 491     cur = reader('0sure.dat.gz')
 492     for x in check_comparer(prior, cur).run():
 493         print x
 494
 495 def main(argv):
 496     if len(argv) != 1:
 497         usage()
 498     if argv[0] == 'scan':
 499         fresh_scan()
 500     elif argv[0] == 'update':
 501         update()
 502     elif argv[0] == 'check':
 503         check_scan()
 504     elif argv[0] == 'signoff':
 505         signoff()
 506     elif argv[0] == 'show':
 507         for i in reader('0sure.dat.gz'):
 508             print i
 509
 510 def usage():
 511     print "Usage: asure {scan|update|check}"
 512     sys.exit(1)
 513
 514 if __name__ == '__main__':
 515     "Test this"
 516     main(sys.argv[1:])