lib/urlwatch/handler.py

   1 #!/usr/bin/python
   2 # -*- coding: utf-8 -*-
   3 #
   4 # urlwatch is a minimalistic URL watcher written in Python
   5 #
   6 # Copyright (c) 2008-2010 Thomas Perl <thp@thpinfo.com>
   7 # All rights reserved.
   8 #
   9 # Redistribution and use in source and binary forms, with or without
  10 # modification, are permitted provided that the following conditions
  11 # are met:
  12 # 1. Redistributions of source code must retain the above copyright
  13 #    notice, this list of conditions and the following disclaimer.
  14 # 2. Redistributions in binary form must reproduce the above copyright
  15 #    notice, this list of conditions and the following disclaimer in the
  16 #    documentation and/or other materials provided with the distribution.
  17 # 3. The name of the author may not be used to endorse or promote products
  18 #    derived from this software without specific prior written permission.
  19 #
  20 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  21 # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  22 # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  23 # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  24 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  25 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  29 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30 #
  31
  32 try:
  33     # Available in Python 2.5 and above and preferred if available
  34     import hashlib
  35     have_hashlib = True
  36 except ImportError:
  37     # "sha" is deprecated since Python 2.5 (throws a warning in Python 2.6)
  38     # Thanks to Frank Palvölgyi for reporting the warning in Python 2.6
  39     import sha
  40     have_hashlib = False
  41
  42 import subprocess
  43 import email.Utils
  44 import urllib2
  45 import os
  46 import stat
  47 import sys
  48 import re
  49
  50 class JobBase(object):
  51     def __init__(self, location):
  52         self.location = location
  53
  54     def __str__(self):
  55         return self.location
  56
  57     def get_guid(self):
  58         if have_hashlib:
  59             sha_hash = hashlib.new('sha1')
  60             sha_hash.update(self.location)
  61             return sha_hash.hexdigest()
  62         else:
  63             return sha.new(self.location).hexdigest()
  64
  65     def retrieve(self, timestamp=None, filter=None, headers=None):
  66         raise Exception('Not implemented')
  67
  68 class ShellError(Exception):
  69     """Exception for shell commands with non-zero exit code"""
  70
  71     def __init__(self, result):
  72         Exception.__init__(self)
  73         self.result = result
  74
  75     def __str__(self):
  76         return '%s: Exit status %d' % (self.__class__.__name__, self.result)
  77
  78 class ShellJob(JobBase):
  79     def retrieve(self, timestamp=None, filter=None, headers=None):
  80         process = subprocess.Popen(self.location, \
  81                 stdout=subprocess.PIPE, \
  82                 shell=True)
  83         stdout_data, stderr_data = process.communicate()
  84         result = process.wait()
  85         if result != 0:
  86             raise ShellError(result)
  87         return filter(self.location, stdout_data)
  88
  89 class UrlJob(JobBase):
  90     CHARSET_RE = re.compile('text/(html|plain); charset=(.*)')
  91
  92     def retrieve(self, timestamp=None, filter=None, headers=None):
  93         headers = dict(headers)
  94         if timestamp is not None:
  95             timestamp = email.Utils.formatdate(timestamp)
  96             headers['If-Modified-Since'] = timestamp
  97         request = urllib2.Request(self.location, None, headers)
  98         response = urllib2.urlopen(request)
  99         headers = response.info()
 100         content = response.read()
 101         encoding = None
 102
 103         # Determine content type via HTTP headers
 104         content_type = headers.get('Content-type', '')
 105         content_type_match = self.CHARSET_RE.match(content_type)
 106         if content_type_match:
 107             encoding = content_type_match.group(2)
 108
 109         if encoding is not None:
 110             # Convert from specified encoding to utf-8
 111             content_unicode = content.decode(encoding, 'ignore')
 112             content = content_unicode.encode('utf-8')
 113
 114         data = filter(self.location, content)
 115         return data
 116
 117
 118 def parse_urls_txt(urls_txt):
 119     jobs = []
 120
 121     # Security checks for shell jobs - only execute if the current UID
 122     # is the same as the file/directory owner and only owner can write
 123     allow_shelljobs = True
 124     shelljob_errors = []
 125     current_uid = os.getuid()
 126
 127     dirname = os.path.dirname(urls_txt)
 128     dir_st = os.stat(dirname)
 129     if (dir_st.st_mode & (stat.S_IWGRP | stat.S_IWOTH)) != 0:
 130         shelljob_errors.append('%s is group/world-writable' % dirname)
 131         allow_shelljobs = False
 132     if dir_st.st_uid != current_uid:
 133         shelljob_errors.append('%s not owned by %s' % (dirname, os.getlogin()))
 134         allow_shelljobs = False
 135
 136     file_st = os.stat(urls_txt)
 137     if (file_st.st_mode & (stat.S_IWGRP | stat.S_IWOTH)) != 0:
 138         shelljob_errors.append('%s is group/world-writable' % urls_txt)
 139         allow_shelljobs = False
 140     if file_st.st_uid != current_uid:
 141         shelljob_errors.append('%s not owned by %s' % (urls_txt, os.getlogin()))
 142         allow_shelljobs = False
 143
 144     for line in open(urls_txt).read().splitlines():
 145         if line.strip().startswith('#') or line.strip() == '':
 146             continue
 147
 148         if line.startswith('|'):
 149             if allow_shelljobs:
 150                 jobs.append(ShellJob(line[1:]))
 151             else:
 152                 print >>sys.stderr, '\n  SECURITY WARNING - Cannot run shell jobs:\n'
 153                 for error in shelljob_errors:
 154                     print >>sys.stderr, '    ', error
 155                 print >>sys.stderr, '\n  Please remove shell jobs or fix these problems.\n'
 156                 sys.exit(1)
 157         else:
 158             jobs.append(UrlJob(line))
 159
 160     return jobs
 161