lib/urlwatch/handler.py

   1 #!/usr/bin/python
   2 # -*- coding: utf-8 -*-
   3 #
   4 # urlwatch is a minimalistic URL watcher written in Python
   5 #
   6 # Copyright (c) 2008-2011 Thomas Perl <thp.io/about>
   7 # All rights reserved.
   8 #
   9 # Redistribution and use in source and binary forms, with or without
  10 # modification, are permitted provided that the following conditions
  11 # are met:
  12 # 1. Redistributions of source code must retain the above copyright
  13 #    notice, this list of conditions and the following disclaimer.
  14 # 2. Redistributions in binary form must reproduce the above copyright
  15 #    notice, this list of conditions and the following disclaimer in the
  16 #    documentation and/or other materials provided with the distribution.
  17 # 3. The name of the author may not be used to endorse or promote products
  18 #    derived from this software without specific prior written permission.
  19 #
  20 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  21 # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  22 # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  23 # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  24 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  25 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  29 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30 #
  31
  32 try:
  33     # Available in Python 2.5 and above and preferred if available
  34     import hashlib
  35     have_hashlib = True
  36 except ImportError:
  37     # "sha" is deprecated since Python 2.5 (throws a warning in Python 2.6)
  38     # Thanks to Frank Palvölgyi for reporting the warning in Python 2.6
  39     import sha
  40     have_hashlib = False
  41
  42 import subprocess
  43 import email.Utils
  44 import urllib2
  45 import os
  46 import stat
  47 import sys
  48 import re
  49
  50 class JobBase(object):
  51     def __init__(self, location):
  52         self.location = location
  53
  54     def __str__(self):
  55         return self.location
  56
  57     def get_guid(self):
  58         if have_hashlib:
  59             sha_hash = hashlib.new('sha1')
  60             sha_hash.update(self.location)
  61             return sha_hash.hexdigest()
  62         else:
  63             return sha.new(self.location).hexdigest()
  64
  65     def retrieve(self, timestamp=None, filter=None, headers=None, log=None):
  66         raise Exception('Not implemented')
  67
  68 class ShellError(Exception):
  69     """Exception for shell commands with non-zero exit code"""
  70
  71     def __init__(self, result):
  72         Exception.__init__(self)
  73         self.result = result
  74
  75     def __str__(self):
  76         return '%s: Exit status %d' % (self.__class__.__name__, self.result)
  77
  78
  79 def use_filter(filter, url, input):
  80     """Apply a filter function to input from an URL"""
  81     output = filter(url, input)
  82
  83     if output is None:
  84         # If the filter does not return a value, it is
  85         # assumed that the input does not need filtering.
  86         # In this case, we simply return the input.
  87         return input
  88
  89     return output
  90
  91
  92 class ShellJob(JobBase):
  93     def retrieve(self, timestamp=None, filter=None, headers=None, log=None):
  94         process = subprocess.Popen(self.location, \
  95                 stdout=subprocess.PIPE, \
  96                 shell=True)
  97         stdout_data, stderr_data = process.communicate()
  98         result = process.wait()
  99         if result != 0:
 100             raise ShellError(result)
 101
 102         return use_filter(filter, self.location, stdout_data)
 103
 104
 105 class UrlJob(JobBase):
 106     CHARSET_RE = re.compile('text/(html|plain); charset=(.*)')
 107
 108     def retrieve(self, timestamp=None, filter=None, headers=None, log=None):
 109         headers = dict(headers)
 110         if timestamp is not None:
 111             timestamp = email.Utils.formatdate(timestamp)
 112             headers['If-Modified-Since'] = timestamp
 113
 114         if ' ' in self.location:
 115             self.location, post_data = self.location.split(' ', 1)
 116             log.info('Sending POST request to %s', self.location)
 117         else:
 118             post_data = None
 119
 120         request = urllib2.Request(self.location, post_data, headers)
 121         response = urllib2.urlopen(request)
 122         headers = response.info()
 123         content = response.read()
 124         encoding = None
 125
 126         # Determine content type via HTTP headers
 127         content_type = headers.get('Content-type', '')
 128         content_type_match = self.CHARSET_RE.match(content_type)
 129         if content_type_match:
 130             encoding = content_type_match.group(2)
 131
 132         if encoding is not None:
 133             # Convert from specified encoding to utf-8
 134             content_unicode = content.decode(encoding, 'ignore')
 135             content = content_unicode.encode('utf-8')
 136
 137         return use_filter(filter, self.location, content)
 138
 139
 140 def parse_urls_txt(urls_txt):
 141     jobs = []
 142
 143     # Security checks for shell jobs - only execute if the current UID
 144     # is the same as the file/directory owner and only owner can write
 145     allow_shelljobs = True
 146     shelljob_errors = []
 147     current_uid = os.getuid()
 148
 149     dirname = os.path.dirname(urls_txt)
 150     dir_st = os.stat(dirname)
 151     if (dir_st.st_mode & (stat.S_IWGRP | stat.S_IWOTH)) != 0:
 152         shelljob_errors.append('%s is group/world-writable' % dirname)
 153         allow_shelljobs = False
 154     if dir_st.st_uid != current_uid:
 155         shelljob_errors.append('%s not owned by %s' % (dirname, os.getlogin()))
 156         allow_shelljobs = False
 157
 158     file_st = os.stat(urls_txt)
 159     if (file_st.st_mode & (stat.S_IWGRP | stat.S_IWOTH)) != 0:
 160         shelljob_errors.append('%s is group/world-writable' % urls_txt)
 161         allow_shelljobs = False
 162     if file_st.st_uid != current_uid:
 163         shelljob_errors.append('%s not owned by %s' % (urls_txt, os.getlogin()))
 164         allow_shelljobs = False
 165
 166     for line in open(urls_txt).read().splitlines():
 167         if line.strip().startswith('#') or line.strip() == '':
 168             continue
 169
 170         if line.startswith('|'):
 171             if allow_shelljobs:
 172                 jobs.append(ShellJob(line[1:]))
 173             else:
 174                 print >>sys.stderr, '\n  SECURITY WARNING - Cannot run shell jobs:\n'
 175                 for error in shelljob_errors:
 176                     print >>sys.stderr, '    ', error
 177                 print >>sys.stderr, '\n  Please remove shell jobs or fix these problems.\n'
 178                 sys.exit(1)
 179         else:
 180             jobs.append(UrlJob(line))
 181
 182     return jobs
 183