2 # -*- coding: utf-8 -*-
4 # urlwatch is a minimalistic URL watcher written in Python
6 # Copyright (c) 2008-2011 Thomas Perl <thp.io/about>
9 # Redistribution and use in source and binary forms, with or without
10 # modification, are permitted provided that the following conditions
12 # 1. Redistributions of source code must retain the above copyright
13 # notice, this list of conditions and the following disclaimer.
14 # 2. Redistributions in binary form must reproduce the above copyright
15 # notice, this list of conditions and the following disclaimer in the
16 # documentation and/or other materials provided with the distribution.
17 # 3. The name of the author may not be used to endorse or promote products
18 # derived from this software without specific prior written permission.
20 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
21 # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
22 # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
23 # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
24 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
25 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
29 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 # Available in Python 2.5 and above and preferred if available
37 # "sha" is deprecated since Python 2.5 (throws a warning in Python 2.6)
38 # Thanks to Frank Palvölgyi for reporting the warning in Python 2.6
50 class JobBase(object):
51 def __init__(self
, location
):
52 self
.location
= location
59 sha_hash
= hashlib
.new('sha1')
60 location
= self
.location
61 if isinstance(location
, unicode):
62 location
= location
.encode('utf-8')
63 sha_hash
.update(location
)
64 return sha_hash
.hexdigest()
66 return sha
.new(self
.location
).hexdigest()
68 def retrieve(self
, timestamp
=None, filter_func
=None, headers
=None,
70 raise Exception('Not implemented')
72 class ShellError(Exception):
73 """Exception for shell commands with non-zero exit code"""
75 def __init__(self
, result
):
76 Exception.__init
__(self
)
80 return '%s: Exit status %d' % (self
.__class
__.__name
__, self
.result
)
83 def use_filter(filter_func
, url
, input):
84 """Apply a filter function to input from an URL"""
85 output
= filter_func(url
, input)
88 # If the filter does not return a value, it is
89 # assumed that the input does not need filtering.
90 # In this case, we simply return the input.
96 class ShellJob(JobBase
):
97 def retrieve(self
, timestamp
=None, filter_func
=None, headers
=None,
99 process
= subprocess
.Popen(self
.location
, \
100 stdout
=subprocess
.PIPE
, \
102 stdout_data
, stderr_data
= process
.communicate()
103 result
= process
.wait()
105 raise ShellError(result
)
107 return use_filter(filter_func
, self
.location
, stdout_data
)
110 class UrlJob(JobBase
):
111 CHARSET_RE
= re
.compile('text/(html|plain); charset=(.*)')
113 def retrieve(self
, timestamp
=None, filter_func
=None, headers
=None,
115 headers
= dict(headers
)
116 if timestamp
is not None:
117 timestamp
= email
.utils
.formatdate(timestamp
)
118 headers
['If-Modified-Since'] = timestamp
120 if ' ' in self
.location
:
121 self
.location
, post_data
= self
.location
.split(' ', 1)
122 log
.info('Sending POST request to %s', self
.location
)
126 request
= urllib2
.Request(self
.location
, post_data
, headers
)
127 response
= urllib2
.urlopen(request
)
128 headers
= response
.info()
129 content
= response
.read()
132 # Determine content type via HTTP headers
133 content_type
= headers
.get('Content-type', '')
134 content_type_match
= self
.CHARSET_RE
.match(content_type
)
135 if content_type_match
:
136 encoding
= content_type_match
.group(2)
138 # Convert from specified encoding to unicode
139 if not isinstance(content
, unicode):
140 content
= content
.decode(encoding
, 'ignore')
142 return use_filter(filter_func
, self
.location
, content
)
145 def parse_urls_txt(urls_txt
):
148 # Security checks for shell jobs - only execute if the current UID
149 # is the same as the file/directory owner and only owner can write
150 allow_shelljobs
= True
152 current_uid
= os
.getuid()
154 dirname
= os
.path
.dirname(urls_txt
)
155 dir_st
= os
.stat(dirname
)
156 if (dir_st
.st_mode
& (stat
.S_IWGRP | stat
.S_IWOTH
)) != 0:
157 shelljob_errors
.append('%s is group/world-writable' % dirname
)
158 allow_shelljobs
= False
159 if dir_st
.st_uid
!= current_uid
:
160 shelljob_errors
.append('%s not owned by %s' % (dirname
, os
.getlogin()))
161 allow_shelljobs
= False
163 file_st
= os
.stat(urls_txt
)
164 if (file_st
.st_mode
& (stat
.S_IWGRP | stat
.S_IWOTH
)) != 0:
165 shelljob_errors
.append('%s is group/world-writable' % urls_txt
)
166 allow_shelljobs
= False
167 if file_st
.st_uid
!= current_uid
:
168 shelljob_errors
.append('%s not owned by %s' % (urls_txt
, os
.getlogin()))
169 allow_shelljobs
= False
171 for line
in open(urls_txt
).read().splitlines():
172 if line
.strip().startswith('#') or line
.strip() == '':
175 if line
.startswith('|'):
177 jobs
.append(ShellJob(line
[1:]))
179 print >>sys
.stderr
, '\n SECURITY WARNING - Cannot run shell jobs:\n'
180 for error
in shelljob_errors
:
181 print >>sys
.stderr
, ' ', error
182 print >>sys
.stderr
, '\n Please remove shell jobs or fix these problems.\n'
185 jobs
.append(UrlJob(line
))