686f53b961746e06f90b48ffbd1eace3d0529c54
2 # -*- coding: utf-8 -*-
4 # urlwatch is a minimalistic URL watcher written in Python
6 # Copyright (c) 2008-2011 Thomas Perl <thp.io/about>
9 # Redistribution and use in source and binary forms, with or without
10 # modification, are permitted provided that the following conditions
12 # 1. Redistributions of source code must retain the above copyright
13 # notice, this list of conditions and the following disclaimer.
14 # 2. Redistributions in binary form must reproduce the above copyright
15 # notice, this list of conditions and the following disclaimer in the
16 # documentation and/or other materials provided with the distribution.
17 # 3. The name of the author may not be used to endorse or promote products
18 # derived from this software without specific prior written permission.
20 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
21 # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
22 # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
23 # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
24 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
25 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
29 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 # Available in Python 2.5 and above and preferred if available
37 # "sha" is deprecated since Python 2.5 (throws a warning in Python 2.6)
38 # Thanks to Frank Palvölgyi for reporting the warning in Python 2.6
50 class JobBase(object):
51 def __init__(self
, location
):
52 self
.location
= location
59 sha_hash
= hashlib
.new('sha1')
60 sha_hash
.update(self
.location
)
61 return sha_hash
.hexdigest()
63 return sha
.new(self
.location
).hexdigest()
65 def retrieve(self
, timestamp
=None, filter=None, headers
=None):
66 raise Exception('Not implemented')
68 class ShellError(Exception):
69 """Exception for shell commands with non-zero exit code"""
71 def __init__(self
, result
):
72 Exception.__init
__(self
)
76 return '%s: Exit status %d' % (self
.__class
__.__name
__, self
.result
)
79 def use_filter(filter, url
, input):
80 """Apply a filter function to input from an URL"""
81 output
= filter(url
, input)
84 # If the filter does not return a value, it is
85 # assumed that the input does not need filtering.
86 # In this case, we simply return the input.
92 class ShellJob(JobBase
):
93 def retrieve(self
, timestamp
=None, filter=None, headers
=None):
94 process
= subprocess
.Popen(self
.location
, \
95 stdout
=subprocess
.PIPE
, \
97 stdout_data
, stderr_data
= process
.communicate()
98 result
= process
.wait()
100 raise ShellError(result
)
102 return use_filter(filter, self
.location
, stdout_data
)
105 class UrlJob(JobBase
):
106 CHARSET_RE
= re
.compile('text/(html|plain); charset=(.*)')
108 def retrieve(self
, timestamp
=None, filter=None, headers
=None):
109 headers
= dict(headers
)
110 if timestamp
is not None:
111 timestamp
= email
.Utils
.formatdate(timestamp
)
112 headers
['If-Modified-Since'] = timestamp
113 request
= urllib2
.Request(self
.location
, None, headers
)
114 response
= urllib2
.urlopen(request
)
115 headers
= response
.info()
116 content
= response
.read()
119 # Determine content type via HTTP headers
120 content_type
= headers
.get('Content-type', '')
121 content_type_match
= self
.CHARSET_RE
.match(content_type
)
122 if content_type_match
:
123 encoding
= content_type_match
.group(2)
125 if encoding
is not None:
126 # Convert from specified encoding to utf-8
127 content_unicode
= content
.decode(encoding
, 'ignore')
128 content
= content_unicode
.encode('utf-8')
130 return use_filter(filter, self
.location
, content
)
133 def parse_urls_txt(urls_txt
):
136 # Security checks for shell jobs - only execute if the current UID
137 # is the same as the file/directory owner and only owner can write
138 allow_shelljobs
= True
140 current_uid
= os
.getuid()
142 dirname
= os
.path
.dirname(urls_txt
)
143 dir_st
= os
.stat(dirname
)
144 if (dir_st
.st_mode
& (stat
.S_IWGRP | stat
.S_IWOTH
)) != 0:
145 shelljob_errors
.append('%s is group/world-writable' % dirname
)
146 allow_shelljobs
= False
147 if dir_st
.st_uid
!= current_uid
:
148 shelljob_errors
.append('%s not owned by %s' % (dirname
, os
.getlogin()))
149 allow_shelljobs
= False
151 file_st
= os
.stat(urls_txt
)
152 if (file_st
.st_mode
& (stat
.S_IWGRP | stat
.S_IWOTH
)) != 0:
153 shelljob_errors
.append('%s is group/world-writable' % urls_txt
)
154 allow_shelljobs
= False
155 if file_st
.st_uid
!= current_uid
:
156 shelljob_errors
.append('%s not owned by %s' % (urls_txt
, os
.getlogin()))
157 allow_shelljobs
= False
159 for line
in open(urls_txt
).read().splitlines():
160 if line
.strip().startswith('#') or line
.strip() == '':
163 if line
.startswith('|'):
165 jobs
.append(ShellJob(line
[1:]))
167 print >>sys
.stderr
, '\n SECURITY WARNING - Cannot run shell jobs:\n'
168 for error
in shelljob_errors
:
169 print >>sys
.stderr
, ' ', error
170 print >>sys
.stderr
, '\n Please remove shell jobs or fix these problems.\n'
173 jobs
.append(UrlJob(line
))