2 Manages the interface cache.
4 @var iface_cache: A singleton cache object. You should normally use this rather than
5 creating new cache objects.
7 # Copyright (C) 2006, Thomas Leonard
8 # See the README file for details, or visit http://0install.net.
12 # We need to know the modification time of each interface, because we refuse
13 # to update to an older version (this prevents an attack where the attacker
14 # sends back an old version which is correctly signed but has a known bug).
16 # The way we store this is a bit complicated due to backward compatibility:
18 # - GPG-signed interfaces have their signatures removed and a last-modified
19 # attribute is stored containing the date from the signature.
21 # - XML-signed interfaces are stored unmodified with their signatures. The
22 # date is extracted from the signature when needed.
24 # - Older versions used to add the last-modified attribute even to files
25 # with XML signatures - these files therefore have invalid signatures and
26 # we extract from the attribute for these.
28 # Eventually, support for the first and third cases will be removed.
31 from logging
import debug
, info
, warn
32 from cStringIO
import StringIO
34 from zeroinstall
.injector
import reader
, basedir
, model
35 from zeroinstall
.injector
.namespaces
import *
36 from zeroinstall
.injector
.model
import *
37 from zeroinstall
import zerostore
40 assert isinstance(t
, (int, long))
41 return time
.strftime('%Y-%m-%d %H:%M:%S UTC', time
.localtime(t
))
43 class PendingFeed(object):
44 """A feed that has been downloaded but not yet added to the interface cache.
45 Feeds remain in this state until the user confirms that they trust at least
46 one of the signatures.
47 @ivar url: URL for the feed
49 @ivar signed_data: the untrusted data
50 @type signed_data: stream
51 @ivar sigs: signatures extracted from signed_data
52 @type sigs: [L{gpg.Signature}]
53 @ivar new_xml: the payload of the signed_data, or the whole thing if XML
56 __slots__
= ['url', 'signed_data', 'sigs', 'new_xml', 'downloads', 'download_callback']
58 def __init__(self
, url
, signed_data
):
59 """Downloaded data is a GPG-signed message.
60 @param url: the URL of the downloaded feed
62 @param signed_data: the downloaded data (not yet trusted)
63 @type signed_data: stream
64 @raise SafeException: if the data is not signed, and logs the actual data"""
66 self
.signed_data
= signed_data
70 def begin_key_downloads(self
, handler
, callback
):
71 """Start downloading any required GPG keys not already on our keyring.
72 When all downloads are done (successful or otherwise), add any new keys
73 to the keyring, L{recheck}, and invoke the callback.
74 If we are already downloading, return and do nothing else.
75 Otherwise, if nothing needs to be downloaded, the callback is invoked immediately.
76 @param handler: handler to manage the download
77 @type handler: L{handler.Handler}
78 @param callback: callback to invoke when done
79 @type callback: function()
85 self
.download_callback
= callback
91 key_url
= urlparse
.urljoin(self
.url
, '%s.gpg' % key_id
)
92 info("Fetching key from %s", key_url
)
93 dl
= handler
.get_download(key_url
)
94 self
.downloads
.append(dl
)
95 dl
.on_success
.append(lambda stream
: self
._downloaded
_key
(dl
, stream
))
97 if not self
.downloads
:
98 self
.download_callback()
100 def _downloaded_key(self
, dl
, stream
):
101 import shutil
, tempfile
102 from zeroinstall
.injector
import gpg
104 self
.downloads
.remove(dl
)
106 info("Importing key for feed '%s'", self
.url
)
108 # Python2.4: can't call fileno() on stream, so save to tmp file instead
109 tmpfile
= tempfile
.TemporaryFile(prefix
= 'injector-dl-data-')
111 shutil
.copyfileobj(stream
, tmpfile
)
116 gpg
.import_key(tmpfile
)
117 except Exception, ex
:
118 warn("Failed to import key for '%s': %s", self
.url
, str(ex
))
122 if not self
.downloads
:
125 self
.download_callback()
128 """Set new_xml and sigs by reading signed_data.
129 You need to call this when previously-missing keys are added to the GPG keyring."""
132 self
.signed_data
.seek(0)
133 stream
, sigs
= gpg
.check_stream(self
.signed_data
)
137 if stream
is not self
.signed_data
:
143 self
.signed_data
.seek(0)
144 info("Failed to check GPG signature. Data received was:\n" + `self
.signed_data
.read()`
)
147 class IfaceCache(object):
149 The interface cache stores downloaded and verified interfaces in
150 ~/.cache/0install.net/interfaces (by default).
152 There are methods to query the cache, add to it, check signatures, etc.
154 When updating the cache, the normal sequence is as follows:
156 1. When the data arrives, L{add_pending} is called.
157 2. Later, L{policy.Policy.process_pending} notices the pending feed and starts processing it.
158 3. It checks the signatures using L{PendingFeed.sigs}.
159 4. If any required GPG keys are missing, L{download_key} is used to fetch
161 5. If none of the keys are trusted, L{handler.Handler.confirm_trust_keys} is called.
162 6. L{update_interface_if_trusted} is called to update the cache.
164 Whenever something needs to be done before the feed can move from the pending
165 state, the process is resumed after the required activity by calling L{policy.Policy.process_pending}.
167 @ivar watchers: objects requiring notification of cache changes.
168 @ivar pending: downloaded feeds which are not yet trusted
169 @type pending: str -> PendingFeed
170 @see: L{iface_cache} - the singleton IfaceCache instance.
173 __slots__
= ['watchers', '_interfaces', 'stores', 'pending']
177 self
._interfaces
= {}
180 self
.stores
= zerostore
.Stores()
182 def add_watcher(self
, w
):
183 """Call C{w.interface_changed(iface)} each time L{update_interface_from_network}
184 changes an interface in the cache."""
185 assert w
not in self
.watchers
186 self
.watchers
.append(w
)
188 def add_pending(self
, pending
):
189 """Add a PendingFeed to the pending dict.
190 @param pending: the untrusted download to add
191 @type pending: PendingFeed
193 assert isinstance(pending
, PendingFeed
)
194 self
.pending
[pending
.url
] = pending
196 def update_interface_if_trusted(self
, interface
, sigs
, xml
):
197 """Update a cached interface (using L{update_interface_from_network})
198 if we trust the signatures, and remove it from L{pending}.
199 If we don't trust any of the signatures, do nothing.
200 @param interface: the interface being updated
201 @type interface: L{model.Interface}
202 @param sigs: signatures from L{gpg.check_stream}
203 @type sigs: [L{gpg.Signature}]
204 @param xml: the downloaded replacement interface document
206 @return: True if the interface was updated
208 @precondition: call L{add_pending}
211 updated
= self
._oldest
_trusted
(sigs
, trust
.domain_from_url(interface
.uri
))
212 if updated
is None: return False # None are trusted
214 if interface
.uri
in self
.pending
:
215 del self
.pending
[interface
.uri
]
217 raise Exception("update_interface_if_trusted, but '%s' not pending!" % interface
.uri
)
219 self
.update_interface_from_network(interface
, xml
, updated
)
222 def download_key(self
, interface
, key_id
):
223 """Download a GPG key.
224 The location of the key is calculated from the uri of the interface.
225 @param interface: the interface which needs the key
226 @param key_id: the GPG long id of the key
227 @todo: This method blocks. It should start a download and return.
228 @deprecated: see PendingFeed
232 import urlparse
, urllib2
, shutil
, tempfile
233 key_url
= urlparse
.urljoin(interface
.uri
, '%s.gpg' % key_id
)
234 info("Fetching key from %s", key_url
)
236 stream
= urllib2
.urlopen(key_url
)
237 # Python2.4: can't call fileno() on stream, so save to tmp file instead
238 tmpfile
= tempfile
.TemporaryFile(prefix
= 'injector-dl-data-')
239 shutil
.copyfileobj(stream
, tmpfile
)
242 except Exception, ex
:
243 raise SafeException("Failed to download key from '%s': %s" % (key_url
, str(ex
)))
248 gpg
.import_key(tmpfile
)
251 def update_interface_from_network(self
, interface
, new_xml
, modified_time
):
252 """Update a cached interface.
253 Called by L{update_interface_if_trusted} if we trust this data.
254 After a successful update, L{writer} is used to update the interface's
255 last_checked time and then all the L{watchers} are notified.
256 @param interface: the interface being updated
257 @type interface: L{model.Interface}
258 @param new_xml: the downloaded replacement interface document
260 @param modified_time: the timestamp of the oldest trusted signature
261 (used as an approximation to the interface's modification time)
262 @type modified_time: long
263 @raises SafeException: if modified_time is older than the currently cached time
265 debug("Updating '%s' from network; modified at %s" %
266 (interface
.name
or interface
.uri
, _pretty_time(modified_time
)))
268 if '\n<!-- Base64 Signature' not in new_xml
:
269 # Only do this for old-style interfaces without
270 # signatures Otherwise, we can get the time from the
271 # signature, and adding this attribute just makes the
273 from xml
.dom
import minidom
274 doc
= minidom
.parseString(new_xml
)
275 doc
.documentElement
.setAttribute('last-modified', str(modified_time
))
277 doc
.writexml(new_xml
)
278 new_xml
= new_xml
.getvalue()
280 self
._import
_new
_interface
(interface
, new_xml
, modified_time
)
283 interface
._main
_feed
.last_checked
= long(time
.time())
284 writer
.save_interface(interface
)
286 info("Updated interface cache entry for %s (modified %s)",
287 interface
.get_name(), _pretty_time(modified_time
))
289 for w
in self
.watchers
:
290 w
.interface_changed(interface
)
292 def _import_new_interface(self
, interface
, new_xml
, modified_time
):
293 """Write new_xml into the cache.
294 @param interface: updated once the new XML is written
295 @param new_xml: the data to write
296 @param modified_time: when new_xml was modified
297 @raises SafeException: if the new mtime is older than the current one
301 upstream_dir
= basedir
.save_cache_path(config_site
, 'interfaces')
302 cached
= os
.path
.join(upstream_dir
, escape(interface
.uri
))
304 if os
.path
.exists(cached
):
305 old_xml
= file(cached
).read()
306 if old_xml
== new_xml
:
310 stream
= file(cached
+ '.new', 'w')
311 stream
.write(new_xml
)
313 os
.utime(cached
+ '.new', (modified_time
, modified_time
))
314 new_mtime
= reader
.check_readable(interface
.uri
, cached
+ '.new')
315 assert new_mtime
== modified_time
317 old_modified
= self
._get
_signature
_date
(interface
.uri
)
318 if old_modified
is None:
319 old_modified
= interface
.last_modified
322 if new_mtime
< old_modified
:
323 raise SafeException("New interface's modification time is before old "
325 "\nOld time: " + _pretty_time(old_modified
) +
326 "\nNew time: " + _pretty_time(new_mtime
) +
327 "\nRefusing update (leaving new copy as " +
329 if new_mtime
== old_modified
:
330 # You used to have to update the modification time manually.
331 # Now it comes from the signature, this check isn't useful
332 # and often causes problems when the stored format changes
333 # (e.g., when we stopped writing last-modified attributes)
335 #raise SafeException("Interface has changed, but modification time "
336 # "hasn't! Refusing update.")
337 os
.rename(cached
+ '.new', cached
)
338 debug("Saved as " + cached
)
340 reader
.update_from_cache(interface
)
342 def get_interface(self
, uri
):
343 """Get the interface for uri, creating a new one if required.
344 New interfaces are initialised from the disk cache, but not from
346 @param uri: the URI of the interface to find
347 @rtype: L{model.Interface}
351 assert isinstance(uri
, unicode)
353 if uri
in self
._interfaces
:
354 return self
._interfaces
[uri
]
356 debug("Initialising new interface object for %s", uri
)
357 self
._interfaces
[uri
] = Interface(uri
)
358 reader
.update_from_cache(self
._interfaces
[uri
])
359 return self
._interfaces
[uri
]
361 def list_all_interfaces(self
):
362 """List all interfaces in the cache.
366 for d
in basedir
.load_cache_paths(config_site
, 'interfaces'):
367 for leaf
in os
.listdir(d
):
368 if not leaf
.startswith('.'):
369 all
.add(unescape(leaf
))
370 for d
in basedir
.load_config_paths(config_site
, config_prog
, 'user_overrides'):
371 for leaf
in os
.listdir(d
):
372 if not leaf
.startswith('.'):
373 all
.add(unescape(leaf
))
374 return list(all
) # Why not just return the set?
376 def add_to_cache(self
, source
, data
):
377 """Add an implementation to the cache.
378 @param source: information about the archive
379 @type source: L{model.DownloadSource}
380 @param data: the data stream
382 @see: L{zerostore.Stores.add_archive_to_cache}
384 assert isinstance(source
, DownloadSource
)
385 required_digest
= source
.implementation
.id
387 self
.stores
.add_archive_to_cache(required_digest
, data
, source
.url
, source
.extract
,
388 type = source
.type, start_offset
= source
.start_offset
or 0)
390 def get_icon_path(self
, iface
):
391 """Get the path of a cached icon for an interface.
392 @param iface: interface whose icon we want
393 @return: the path of the cached icon, or None if not cached.
395 return basedir
.load_first_cache(config_site
, 'interface_icons',
398 def get_cached_signatures(self
, uri
):
399 """Verify the cached interface using GPG.
400 Only new-style XML-signed interfaces retain their signatures in the cache.
401 @param uri: the feed to check
403 @return: a list of signatures, or None
404 @rtype: [L{gpg.Signature}] or None
407 if uri
.startswith('/'):
410 old_iface
= basedir
.load_first_cache(config_site
, 'interfaces', escape(uri
))
411 if old_iface
is None:
414 return gpg
.check_stream(file(old_iface
))[1]
415 except SafeException
, ex
:
416 debug("No signatures (old-style interface): %s" % ex
)
419 def _get_signature_date(self
, uri
):
420 """Read the date-stamp from the signature of the cached interface.
421 If the date-stamp is unavailable, returns None."""
423 sigs
= self
.get_cached_signatures(uri
)
425 return self
._oldest
_trusted
(sigs
, trust
.domain_from_url(uri
))
427 def _oldest_trusted(self
, sigs
, domain
):
428 """Return the date of the oldest trusted signature in the list, or None if there
429 are no trusted sigs in the list."""
430 trusted
= [s
.get_timestamp() for s
in sigs
if s
.is_trusted(domain
)]
435 def mark_as_checking(self
, url
):
436 """Touch a 'last_check_attempt_timestamp' file for this feed.
437 If url is a local path, nothing happens.
438 This prevents us from repeatedly trying to download a failing feed many
439 times in a short period."""
440 if url
.startswith('/'):
442 feeds_dir
= basedir
.save_cache_path(config_site
, config_prog
, 'last-check-attempt')
443 timestamp_path
= os
.path
.join(feeds_dir
, model
._pretty
_escape
(url
))
444 fd
= os
.open(timestamp_path
, os
.O_WRONLY | os
.O_CREAT
, 0644)
447 def get_last_check_attempt(self
, url
):
448 """Return the time of the most recent update attempt for a feed.
449 @see: L{mark_as_checking}
450 @return: The time, or None if none is recorded"""
451 timestamp_path
= basedir
.load_first_cache(config_site
, config_prog
, 'last-check-attempt', model
._pretty
_escape
(url
))
453 return os
.stat(timestamp_path
).st_mtime
456 iface_cache
= IfaceCache()