lib/urlwatch/handler.py

   1 #!/usr/bin/python
   2 # -*- coding: utf-8 -*-
   3 #
   4 # urlwatch is a minimalistic URL watcher written in Python
   5 #
   6 # Copyright (c) 2008-2011 Thomas Perl <thp.io/about>
   7 # All rights reserved.
   8 #
   9 # Redistribution and use in source and binary forms, with or without
  10 # modification, are permitted provided that the following conditions
  11 # are met:
  12 # 1. Redistributions of source code must retain the above copyright
  13 #    notice, this list of conditions and the following disclaimer.
  14 # 2. Redistributions in binary form must reproduce the above copyright
  15 #    notice, this list of conditions and the following disclaimer in the
  16 #    documentation and/or other materials provided with the distribution.
  17 # 3. The name of the author may not be used to endorse or promote products
  18 #    derived from this software without specific prior written permission.
  19 #
  20 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  21 # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  22 # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  23 # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  24 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  25 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  29 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30 #
  31
  32 try:
  33     # Available in Python 2.5 and above and preferred if available
  34     import hashlib
  35     have_hashlib = True
  36 except ImportError:
  37     # "sha" is deprecated since Python 2.5 (throws a warning in Python 2.6)
  38     # Thanks to Frank Palvölgyi for reporting the warning in Python 2.6
  39     import sha
  40     have_hashlib = False
  41
  42 import subprocess
  43 import email.utils
  44 import urllib2
  45 import os
  46 import stat
  47 import sys
  48 import re
  49
  50 class JobBase(object):
  51     def __init__(self, location):
  52         self.location = location
  53
  54     def __str__(self):
  55         return self.location
  56
  57     def get_guid(self):
  58         if have_hashlib:
  59             sha_hash = hashlib.new('sha1')
  60             location = self.location
  61             if isinstance(location, unicode):
  62                 location = location.encode('utf-8')
  63             sha_hash.update(location)
  64             return sha_hash.hexdigest()
  65         else:
  66             return sha.new(self.location).hexdigest()
  67
  68     def retrieve(self, timestamp=None, filter_func=None, headers=None,
  69             log=None):
  70         raise Exception('Not implemented')
  71
  72 class ShellError(Exception):
  73     """Exception for shell commands with non-zero exit code"""
  74
  75     def __init__(self, result):
  76         Exception.__init__(self)
  77         self.result = result
  78
  79     def __str__(self):
  80         return '%s: Exit status %d' % (self.__class__.__name__, self.result)
  81
  82
  83 def use_filter(filter_func, url, input):
  84     """Apply a filter function to input from an URL"""
  85     output = filter_func(url, input)
  86
  87     if output is None:
  88         # If the filter does not return a value, it is
  89         # assumed that the input does not need filtering.
  90         # In this case, we simply return the input.
  91         return input
  92
  93     return output
  94
  95
  96 class ShellJob(JobBase):
  97     def retrieve(self, timestamp=None, filter_func=None, headers=None,
  98             log=None):
  99         process = subprocess.Popen(self.location, \
 100                 stdout=subprocess.PIPE, \
 101                 shell=True)
 102         stdout_data, stderr_data = process.communicate()
 103         result = process.wait()
 104         if result != 0:
 105             raise ShellError(result)
 106
 107         return use_filter(filter_func, self.location, stdout_data)
 108
 109
 110 class UrlJob(JobBase):
 111     CHARSET_RE = re.compile('text/(html|plain); charset=(.*)')
 112
 113     def retrieve(self, timestamp=None, filter_func=None, headers=None,
 114             log=None):
 115         headers = dict(headers)
 116         if timestamp is not None:
 117             timestamp = email.utils.formatdate(timestamp)
 118             headers['If-Modified-Since'] = timestamp
 119
 120         if ' ' in self.location:
 121             self.location, post_data = self.location.split(' ', 1)
 122             log.info('Sending POST request to %s', self.location)
 123         else:
 124             post_data = None
 125
 126         request = urllib2.Request(self.location, post_data, headers)
 127         response = urllib2.urlopen(request)
 128         headers = response.info()
 129         content = response.read()
 130         encoding = 'utf-8'
 131
 132         # Determine content type via HTTP headers
 133         content_type = headers.get('Content-type', '')
 134         content_type_match = self.CHARSET_RE.match(content_type)
 135         if content_type_match:
 136             encoding = content_type_match.group(2)
 137
 138         # Convert from specified encoding to unicode
 139         if not isinstance(content, unicode):
 140             content = content.decode(encoding, 'ignore')
 141
 142         return use_filter(filter_func, self.location, content)
 143
 144
 145 def parse_urls_txt(urls_txt):
 146     jobs = []
 147
 148     # Security checks for shell jobs - only execute if the current UID
 149     # is the same as the file/directory owner and only owner can write
 150     allow_shelljobs = True
 151     shelljob_errors = []
 152     current_uid = os.getuid()
 153
 154     dirname = os.path.dirname(urls_txt)
 155     dir_st = os.stat(dirname)
 156     if (dir_st.st_mode & (stat.S_IWGRP | stat.S_IWOTH)) != 0:
 157         shelljob_errors.append('%s is group/world-writable' % dirname)
 158         allow_shelljobs = False
 159     if dir_st.st_uid != current_uid:
 160         shelljob_errors.append('%s not owned by %s' % (dirname, os.getlogin()))
 161         allow_shelljobs = False
 162
 163     file_st = os.stat(urls_txt)
 164     if (file_st.st_mode & (stat.S_IWGRP | stat.S_IWOTH)) != 0:
 165         shelljob_errors.append('%s is group/world-writable' % urls_txt)
 166         allow_shelljobs = False
 167     if file_st.st_uid != current_uid:
 168         shelljob_errors.append('%s not owned by %s' % (urls_txt, os.getlogin()))
 169         allow_shelljobs = False
 170
 171     for line in open(urls_txt).read().splitlines():
 172         if line.strip().startswith('#') or line.strip() == '':
 173             continue
 174
 175         if line.startswith('|'):
 176             if allow_shelljobs:
 177                 jobs.append(ShellJob(line[1:]))
 178             else:
 179                 print >>sys.stderr, '\n  SECURITY WARNING - Cannot run shell jobs:\n'
 180                 for error in shelljob_errors:
 181                     print >>sys.stderr, '    ', error
 182                 print >>sys.stderr, '\n  Please remove shell jobs or fix these problems.\n'
 183                 sys.exit(1)
 184         else:
 185             jobs.append(UrlJob(line))
 186
 187     return jobs
 188