1 # Copyright (C) 2010, Thomas Leonard
2 # See the COPYING file for details, or visit http://0install.net.
4 import os
, time
, codecs
5 import xml
.etree
.ElementTree
as ET
7 from zeroinstall
.injector
.iface_cache
import iface_cache
8 from zeroinstall
.injector
import gpg
, trust
, namespaces
, model
, qdom
9 from zeroinstall
.support
import basedir
11 from support
import ensure_dirs
, get_feed_dir
13 def format_date(date
):
14 return time
.strftime("%Y-%m-%d", time
.gmtime(date
))
16 # When people change keys, add a mapping so that their new feeds appear under the same user
17 # TODO: this should be site configuration
19 # New key Original key
20 '617794D7C3DFE0FFF572065C0529FDB71FB13910' : '92429807C9853C0744A68B9AAE07828059A53CC1',
21 '6AD4A9C482F1D3F537C0354FC8CC44742B11FF89' : 'FD3208AD535F2B63BCEDB2BFFB013BAB74FFF135',
22 '1DFE86921CBA7BCB691DA2434F5A1693E18E1E91' : '0C5C7BC77B70E7BA813478B6FF29FF60ACB8DFE8',
23 '2E2B4E59CAC8D874CD2759D34B1095AF2E992B19' : 'C82D382AAB381A54529019D6A0F9B035686C6996',
24 'DA9825AECAD089757CDABD8E07133F96CA74D8BA' : '92429807C9853C0744A68B9AAE07828059A53CC1',
25 '7722DC5085B903FF176CCAA9695BA303C9839ABC' : '03DC5771716A5A329CA97EA64AB8A8E7613A266F',
26 '39AD3DDE2B988623D7F868591C319390658A683A' : 'D30B76E435BD65448F2A57C7B8E1967CBF45481E',
27 '4CFBD0B5B7102BF66E9F12AEFBCAE33FC2DE322B' : '92429807C9853C0744A68B9AAE07828059A53CC1',
28 'FA2577C515715EEE1261D3B0EFD438E5019F0846' : '7EADC3F1EFE150C371EDE0A15B5CB97421BAA5DC',
31 reverse_aliases
= {} # user ID -> list of their other keys
32 for new
, original
in aliases
.iteritems():
33 if original
not in reverse_aliases
:
34 reverse_aliases
[original
] = []
35 reverse_aliases
[original
].append(new
)
37 # Feeds with these keys must not be mirrored
39 test_keys
.add('5E22F6A13A76F396AC68B5F29B1F5D7F9721DA90')
40 test_keys
.add('2E32123D8BE241A3B6D91E0301685F11607BB2C5')
42 def make_feed_element(parent
, feed
, active
):
43 feed_element
= ET
.SubElement(parent
, 'feed')
44 feed_element
.attrib
['active'] = str(active
)
45 feed_element
.attrib
['local-dir'] = get_feed_dir(feed
.url
).replace('#', '%23')
46 feed_element
.attrib
['url'] = feed
.url
47 feed_element
.attrib
['name'] = feed
.get_name()
48 feed_element
.attrib
['implementations'] = str(count_impls(feed
.url
))
49 feed_element
.attrib
['last-modified'] = format_date(feed
.last_modified
)
50 feed_element
.attrib
['summary'] = feed
.summary
53 if not os
.path
.exists(path
):
55 with
open(path
) as stream
:
58 def write_if_changed(xml
, path
):
60 xml
.write(new
, encoding
='utf-8')
61 if contents(path
) == contents(new
):
69 if url
not in cached_counts
:
70 cached
= basedir
.load_first_cache(namespaces
.config_site
, 'interfaces', model
.escape(url
))
72 with
open(cached
) as stream
:
73 cached_doc
= qdom
.parse(stream
)
76 if elem
.uri
!= namespaces
.XMLNS_IFACE
: return 0
77 if elem
.name
== 'implementation' or elem
.name
== 'package-implementation':
80 for child
in elem
.childNodes
:
83 cached_counts
[url
] = count(cached_doc
)
85 cached_counts
[url
] = 0
86 return cached_counts
[url
]
91 self
.last_active
= None
93 self
.n_implementations
= 0
97 def add_feed(self
, feed
, sig
, active
):
98 assert feed
not in self
.feeds
, feed
99 self
.feeds
[feed
] = active
100 mtime
= sig
.get_timestamp()
101 if self
.last_active
is None or self
.last_active
< mtime
:
102 self
.last_active
= mtime
105 self
.n_implementations
+= count_impls(feed
.url
)
109 def as_xml(self
, user_keys
):
110 root
= ET
.Element('user')
112 name
= ET
.SubElement(root
, 'name')
113 name
.text
= self
.key
.get_short_name()
115 feeds
= ET
.SubElement(root
, 'feeds')
117 sorted_feeds
= sorted([(feed
.get_name().lower(), feed
) for feed
in self
.feeds
.keys()])
118 for unused
, feed
in sorted_feeds
:
119 make_feed_element(feeds
, feed
, self
.feeds
[feed
])
121 stats
= ET
.SubElement(root
, 'stats')
122 stats
.attrib
['feeds'] = str(self
.n_feeds
)
123 stats
.attrib
['implementations'] = str(self
.n_implementations
)
125 stats
.attrib
['inactive_feeds'] = str(self
.n_inactive
)
126 stats
.attrib
['karma'] = str(self
.get_karma())
128 keys
= ET
.SubElement(root
, 'keys')
129 for key
in user_keys
:
130 key_elem
= ET
.SubElement(keys
, 'key')
131 key_elem
.attrib
['name'] = key
.get_short_name()
132 key_elem
.attrib
['fingerprint'] = key
.fingerprint
133 key_elem
.attrib
['keyid'] = key
.fingerprint
[-16:]
135 return ET
.ElementTree(root
)
138 return 10 * self
.n_feeds
+ self
.n_implementations
+ self
.n_inactive
140 def export_users(pairs
):
141 root
= ET
.Element('users')
142 for karma
, user
in pairs
:
143 elem
= ET
.SubElement(root
, "user")
144 elem
.attrib
["name"] = user
.key
.get_short_name()
145 elem
.attrib
["karma"] = str(karma
)
146 elem
.attrib
["uid"] = user
.key
.fingerprint
147 return ET
.ElementTree(root
)
149 def export_sites(tuples
):
150 root
= ET
.Element('sites')
151 for n_feeds
, domain
, feeds
in tuples
:
152 elem
= ET
.SubElement(root
, "site")
153 elem
.attrib
["name"] = domain
154 elem
.attrib
["feeds"] = str(n_feeds
)
155 elem
.attrib
["site-path"] = 'sites/site-%s.html' % domain
156 return ET
.ElementTree(root
)
158 """Keep track of some statistics."""
161 self
.users
= {} # Fingerprint -> User
162 self
.sites
= {} # Domain -> [Feed]
164 self
.active
= {} # Feed -> bool
166 def add_feed(self
, feed
, active
):
167 self
.active
[feed
] = active
169 metadata
= ET
.Element('metadata')
170 metadata
.attrib
["active"] = str(active
)
172 sigs
= iface_cache
.get_cached_signatures(feed
.url
)
174 for sig
in sigs
or []:
175 if isinstance(sig
, gpg
.ValidSig
):
176 fingerprint
= aliases
.get(sig
.fingerprint
, sig
.fingerprint
)
177 assert fingerprint
not in aliases
, fingerprint
179 assert fingerprint
not in test_keys
, (fingerprint
, feed
)
180 if fingerprint
not in self
.users
:
181 self
.users
[fingerprint
] = User()
182 self
.users
[fingerprint
].add_feed(feed
, sig
, active
)
184 signer
= ET
.SubElement(metadata
, "signer")
185 signer
.attrib
["user"] = fingerprint
186 signer
.attrib
["date"] = format_date(sig
.get_timestamp())
188 signer
= ET
.SubElement(metadata
, "signer")
189 signer
.attrib
["error"] = unicode(sig
)
191 domain
= trust
.domain_from_url(feed
.url
)
192 if domain
not in self
.sites
:
193 self
.sites
[domain
] = []
194 self
.sites
[domain
].append(feed
)
196 self
.feeds
.append((feed
, metadata
))
198 def write_summary(self
, topdir
):
200 keys
= gpg
.load_keys(self
.users
.keys() + aliases
.keys())
202 for fingerprint
, user
in self
.users
.iteritems():
203 user
.key
= keys
[fingerprint
]
205 # 0launch <= 0.45 doesn't returns names in unicode
206 unicode(user
.key
.name
)
208 user
.key
.name
= codecs
.decode(user
.key
.name
, 'utf-8')
209 names
.append((user
.key
.name
, fingerprint
))
210 for name
, fingerprint
in sorted(names
):
211 user
= self
.users
[fingerprint
]
212 user_dir
= ensure_dirs(os
.path
.join(topdir
, 'users', fingerprint
))
214 user_keys
= [fingerprint
] + reverse_aliases
.get(fingerprint
, [])
215 user_xml
= user
.as_xml([keys
[k
] for k
in user_keys
])
216 write_if_changed(user_xml
, os
.path
.join(user_dir
, 'user.xml'))
217 top_users
.append((user
.get_karma(), user
))
219 users_xml
= export_users(reversed(sorted(top_users
)))
220 write_if_changed(users_xml
, os
.path
.join(topdir
, 'top-users.xml'))
222 for domain
, feeds
in self
.sites
.iteritems():
223 site
= ET
.Element('site')
224 site
.attrib
["name"] = domain
225 feeds_elem
= ET
.SubElement(site
, "feeds")
226 sorted_feeds
= sorted([(feed
.get_name().lower(), feed
) for feed
in feeds
])
227 for name
, feed
in sorted_feeds
:
228 make_feed_element(feeds_elem
, feed
, self
.active
[feed
])
229 site_xml
= ET
.ElementTree(site
)
230 write_if_changed(site_xml
, os
.path
.join(topdir
, 'sites', 'site-%s.xml' % domain
))
232 top_sites
= [(len(feeds
), domain
, feeds
) for domain
, feeds
in self
.sites
.iteritems()]
233 sites_xml
= export_sites(reversed(sorted(top_sites
)))
234 write_if_changed(sites_xml
, os
.path
.join(topdir
, 'top-sites.xml'))
236 for feed
, metadata
in self
.feeds
:
237 for signer
in metadata
.findall("signer"):
238 if "user" in signer
.attrib
:
239 fingerprint
= signer
.attrib
["user"]
240 user
= self
.users
[fingerprint
]
241 signer
.attrib
["name"] = user
.key
.get_short_name()
243 metadata_xml
= ET
.ElementTree(metadata
)
244 feed_dir
= get_feed_dir(feed
.url
)
245 write_if_changed(metadata_xml
, os
.path
.join(topdir
, feed_dir
, 'metadata.xml'))