2 Manages the feed cache.
4 @var iface_cache: A singleton cache object. You should normally use this rather than
5 creating new cache objects.
7 # Copyright (C) 2008, Thomas Leonard
8 # See the README file for details, or visit http://0install.net.
12 # We need to know the modification time of each interface, because we refuse
13 # to update to an older version (this prevents an attack where the attacker
14 # sends back an old version which is correctly signed but has a known bug).
16 # The way we store this is a bit complicated due to backward compatibility:
18 # - GPG-signed interfaces have their signatures removed and a last-modified
19 # attribute is stored containing the date from the signature.
21 # - XML-signed interfaces are stored unmodified with their signatures. The
22 # date is extracted from the signature when needed.
24 # - Older versions used to add the last-modified attribute even to files
25 # with XML signatures - these files therefore have invalid signatures and
26 # we extract from the attribute for these.
28 # Eventually, support for the first and third cases will be removed.
31 from logging
import debug
, info
, warn
32 from cStringIO
import StringIO
34 from zeroinstall
.support
import basedir
35 from zeroinstall
.injector
import reader
, model
36 from zeroinstall
.injector
.namespaces
import *
37 from zeroinstall
.injector
.model
import *
38 from zeroinstall
import zerostore
41 assert isinstance(t
, (int, long)), t
42 return time
.strftime('%Y-%m-%d %H:%M:%S UTC', time
.localtime(t
))
44 class ReplayAttack(SafeException
):
45 """Attempt to import a feed that's older than the one in the cache."""
48 class PendingFeed(object):
49 """A feed that has been downloaded but not yet added to the interface cache.
50 Feeds remain in this state until the user confirms that they trust at least
51 one of the signatures.
52 @ivar url: URL for the feed
54 @ivar signed_data: the untrusted data
55 @type signed_data: stream
56 @ivar sigs: signatures extracted from signed_data
57 @type sigs: [L{gpg.Signature}]
58 @ivar new_xml: the payload of the signed_data, or the whole thing if XML
61 __slots__
= ['url', 'signed_data', 'sigs', 'new_xml']
63 def __init__(self
, url
, signed_data
):
64 """Downloaded data is a GPG-signed message.
65 @param url: the URL of the downloaded feed
67 @param signed_data: the downloaded data (not yet trusted)
68 @type signed_data: stream
69 @raise SafeException: if the data is not signed, and logs the actual data"""
71 self
.signed_data
= signed_data
74 def download_keys(self
, handler
, feed_hint
= None, key_mirror
= None):
75 """Download any required GPG keys not already on our keyring.
76 When all downloads are done (successful or otherwise), add any new keys
77 to the keyring, L{recheck}.
78 @param handler: handler to manage the download
79 @type handler: L{handler.Handler}
80 @param key_mirror: URL of directory containing keys, or None to use feed's directory
89 key_url
= urlparse
.urljoin(key_mirror
or self
.url
, '%s.gpg' % key_id
)
90 info("Fetching key from %s", key_url
)
91 dl
= handler
.get_download(key_url
, hint
= feed_hint
)
92 downloads
[dl
.downloaded
] = (dl
, dl
.tempfile
)
93 blockers
.append(dl
.downloaded
)
98 from zeroinstall
.support
import tasks
103 old_blockers
= blockers
106 for b
in old_blockers
:
110 dl
, stream
= downloads
[b
]
112 self
._downloaded
_key
(stream
)
117 _
, exception
, tb
= sys
.exc_info()
118 warn("Failed to import key for '%s': %s", self
.url
, str(exception
))
120 if exception
and not any_success
:
121 raise exception
, None, tb
125 def _downloaded_key(self
, stream
):
126 import shutil
, tempfile
127 from zeroinstall
.injector
import gpg
129 info("Importing key for feed '%s'", self
.url
)
131 # Python2.4: can't call fileno() on stream, so save to tmp file instead
132 tmpfile
= tempfile
.TemporaryFile(prefix
= 'injector-dl-data-')
134 shutil
.copyfileobj(stream
, tmpfile
)
138 gpg
.import_key(tmpfile
)
143 """Set new_xml and sigs by reading signed_data.
144 You need to call this when previously-missing keys are added to the GPG keyring."""
147 self
.signed_data
.seek(0)
148 stream
, sigs
= gpg
.check_stream(self
.signed_data
)
152 if stream
is not self
.signed_data
:
158 self
.signed_data
.seek(0)
159 info("Failed to check GPG signature. Data received was:\n" + `self
.signed_data
.read()`
)
162 class IfaceCache(object):
164 The interface cache stores downloaded and verified interfaces in
165 ~/.cache/0install.net/interfaces (by default).
167 There are methods to query the cache, add to it, check signatures, etc.
169 The cache is updated by L{fetch.Fetcher}.
171 Confusingly, this class is really two caches combined: the in-memory
172 cache of L{model.Interface} objects, and an on-disk cache of L{model.ZeroInstallFeed}s.
173 It will probably be split into two in future.
175 @ivar pending: downloaded feeds which are not yet trusted
176 @type pending: str -> PendingFeed
177 @see: L{iface_cache} - the singleton IfaceCache instance.
180 __slots__
= ['_interfaces', 'stores', 'pending']
183 self
._interfaces
= {}
186 self
.stores
= zerostore
.Stores()
188 def add_pending(self
, pending
):
189 """Add a PendingFeed to the pending dict.
190 @param pending: the untrusted download to add
191 @type pending: PendingFeed
193 assert isinstance(pending
, PendingFeed
)
194 self
.pending
[pending
.url
] = pending
196 def update_interface_if_trusted(self
, interface
, sigs
, xml
):
197 """Update a cached interface (using L{update_interface_from_network})
198 if we trust the signatures, and remove it from L{pending}.
199 If we don't trust any of the signatures, do nothing.
200 @param interface: the interface being updated
201 @type interface: L{model.Interface}
202 @param sigs: signatures from L{gpg.check_stream}
203 @type sigs: [L{gpg.Signature}]
204 @param xml: the downloaded replacement interface document
206 @return: True if the interface was updated
208 @precondition: call L{add_pending}
211 updated
= self
._oldest
_trusted
(sigs
, trust
.domain_from_url(interface
.uri
))
212 if updated
is None: return False # None are trusted
214 if interface
.uri
in self
.pending
:
215 del self
.pending
[interface
.uri
]
217 raise Exception("update_interface_if_trusted, but '%s' not pending!" % interface
.uri
)
219 self
.update_interface_from_network(interface
, xml
, updated
)
222 def update_interface_from_network(self
, interface
, new_xml
, modified_time
):
223 """Update a cached interface.
224 Called by L{update_interface_if_trusted} if we trust this data.
225 After a successful update, L{writer} is used to update the interface's
227 @param interface: the interface being updated
228 @type interface: L{model.Interface}
229 @param new_xml: the downloaded replacement interface document
231 @param modified_time: the timestamp of the oldest trusted signature
232 (used as an approximation to the interface's modification time)
233 @type modified_time: long
234 @raises SafeException: if modified_time is older than the currently cached time
236 debug("Updating '%s' from network; modified at %s" %
237 (interface
.name
or interface
.uri
, _pretty_time(modified_time
)))
239 if '\n<!-- Base64 Signature' not in new_xml
:
240 # Only do this for old-style interfaces without
241 # signatures Otherwise, we can get the time from the
242 # signature, and adding this attribute just makes the
244 from xml
.dom
import minidom
245 doc
= minidom
.parseString(new_xml
)
246 doc
.documentElement
.setAttribute('last-modified', str(modified_time
))
248 doc
.writexml(new_xml
)
249 new_xml
= new_xml
.getvalue()
251 self
._import
_new
_interface
(interface
, new_xml
, modified_time
)
254 interface
._main
_feed
.last_checked
= long(time
.time())
255 writer
.save_interface(interface
)
257 info("Updated interface cache entry for %s (modified %s)",
258 interface
.get_name(), _pretty_time(modified_time
))
260 def _import_new_interface(self
, interface
, new_xml
, modified_time
):
261 """Write new_xml into the cache.
262 @param interface: updated once the new XML is written
263 @param new_xml: the data to write
264 @param modified_time: when new_xml was modified
265 @raises SafeException: if the new mtime is older than the current one
269 upstream_dir
= basedir
.save_cache_path(config_site
, 'interfaces')
270 cached
= os
.path
.join(upstream_dir
, escape(interface
.uri
))
272 if os
.path
.exists(cached
):
273 old_xml
= file(cached
).read()
274 if old_xml
== new_xml
:
278 stream
= file(cached
+ '.new', 'w')
279 stream
.write(new_xml
)
281 os
.utime(cached
+ '.new', (modified_time
, modified_time
))
282 new_mtime
= reader
.check_readable(interface
.uri
, cached
+ '.new')
283 assert new_mtime
== modified_time
285 old_modified
= self
._get
_signature
_date
(interface
.uri
)
286 if old_modified
is None:
287 old_modified
= interface
.last_modified
290 if new_mtime
< old_modified
:
291 os
.unlink(cached
+ '.new')
292 raise ReplayAttack("New interface's modification time is before old "
294 "\nOld time: " + _pretty_time(old_modified
) +
295 "\nNew time: " + _pretty_time(new_mtime
) +
296 "\nRefusing update.")
297 if new_mtime
== old_modified
:
298 # You used to have to update the modification time manually.
299 # Now it comes from the signature, this check isn't useful
300 # and often causes problems when the stored format changes
301 # (e.g., when we stopped writing last-modified attributes)
303 #raise SafeException("Interface has changed, but modification time "
304 # "hasn't! Refusing update.")
305 os
.rename(cached
+ '.new', cached
)
306 debug("Saved as " + cached
)
308 reader
.update_from_cache(interface
)
310 def get_feed(self
, url
):
311 """Get a feed from the cache.
312 @param url: the URL of the feed
313 @return: the feed, or None if it isn't cached
314 @rtype: L{model.ZeroInstallFeed}"""
315 # TODO: This isn't a good implementation
316 iface
= self
.get_interface(url
)
317 feed
= iface
._main
_feed
318 if not isinstance(feed
, model
.DummyFeed
):
322 def get_interface(self
, uri
):
323 """Get the interface for uri, creating a new one if required.
324 New interfaces are initialised from the disk cache, but not from
326 @param uri: the URI of the interface to find
327 @rtype: L{model.Interface}
331 assert isinstance(uri
, unicode)
333 if uri
in self
._interfaces
:
334 return self
._interfaces
[uri
]
336 debug("Initialising new interface object for %s", uri
)
337 self
._interfaces
[uri
] = Interface(uri
)
338 reader
.update_from_cache(self
._interfaces
[uri
])
339 return self
._interfaces
[uri
]
341 def list_all_interfaces(self
):
342 """List all interfaces in the cache.
346 for d
in basedir
.load_cache_paths(config_site
, 'interfaces'):
347 for leaf
in os
.listdir(d
):
348 if not leaf
.startswith('.'):
349 all
.add(unescape(leaf
))
350 for d
in basedir
.load_config_paths(config_site
, config_prog
, 'user_overrides'):
351 for leaf
in os
.listdir(d
):
352 if not leaf
.startswith('.'):
353 all
.add(unescape(leaf
))
354 return list(all
) # Why not just return the set?
356 def get_icon_path(self
, iface
):
357 """Get the path of a cached icon for an interface.
358 @param iface: interface whose icon we want
359 @return: the path of the cached icon, or None if not cached.
361 return basedir
.load_first_cache(config_site
, 'interface_icons',
364 def get_cached_signatures(self
, uri
):
365 """Verify the cached interface using GPG.
366 Only new-style XML-signed interfaces retain their signatures in the cache.
367 @param uri: the feed to check
369 @return: a list of signatures, or None
370 @rtype: [L{gpg.Signature}] or None
373 if uri
.startswith('/'):
376 old_iface
= basedir
.load_first_cache(config_site
, 'interfaces', escape(uri
))
377 if old_iface
is None:
380 return gpg
.check_stream(file(old_iface
))[1]
381 except SafeException
, ex
:
382 debug("No signatures (old-style interface): %s" % ex
)
385 def _get_signature_date(self
, uri
):
386 """Read the date-stamp from the signature of the cached interface.
387 If the date-stamp is unavailable, returns None."""
389 sigs
= self
.get_cached_signatures(uri
)
391 return self
._oldest
_trusted
(sigs
, trust
.domain_from_url(uri
))
393 def _oldest_trusted(self
, sigs
, domain
):
394 """Return the date of the oldest trusted signature in the list, or None if there
395 are no trusted sigs in the list."""
396 trusted
= [s
.get_timestamp() for s
in sigs
if s
.is_trusted(domain
)]
401 def mark_as_checking(self
, url
):
402 """Touch a 'last_check_attempt_timestamp' file for this feed.
403 If url is a local path, nothing happens.
404 This prevents us from repeatedly trying to download a failing feed many
405 times in a short period."""
406 if url
.startswith('/'):
408 feeds_dir
= basedir
.save_cache_path(config_site
, config_prog
, 'last-check-attempt')
409 timestamp_path
= os
.path
.join(feeds_dir
, model
._pretty
_escape
(url
))
410 fd
= os
.open(timestamp_path
, os
.O_WRONLY | os
.O_CREAT
, 0644)
413 def get_last_check_attempt(self
, url
):
414 """Return the time of the most recent update attempt for a feed.
415 @see: L{mark_as_checking}
416 @return: The time, or None if none is recorded
417 @rtype: float | None"""
418 timestamp_path
= basedir
.load_first_cache(config_site
, config_prog
, 'last-check-attempt', model
._pretty
_escape
(url
))
420 return os
.stat(timestamp_path
).st_mtime
423 iface_cache
= IfaceCache()