have different dists
[camarabuntu.git] / bin / apt.py
blobf8ee7af9c0f2b769d4fde091a3709e4effe97214
1 import os, re, commands, copy, urllib, gzip, tempfile, operator
3 class Dependency():
4 def __init__(self, string=None, package_name=None, version=None, relation=None):
5 if string is not None:
6 depends_re = re.compile( r"(?P<name>\S*) \((?P<relation><<|<=|=|>=|>>) (?P<version>\S*)\)" )
7 result = depends_re.search( string )
8 if result is None:
9 self.name = string
10 self.version = None
11 self.relation = None
12 else:
13 self.name = result.group('name')
14 self.version = result.group('version')
15 self.relation = result.group('relation')
16 else:
17 self.name = name
18 self.version = version
19 self.relation = relation
21 def __str__(self):
22 if self.relation is None and self.version is None:
23 return self.name
24 else:
25 return "%s (%s %s)" % (self.name, self.relation, self.version)
27 def __repr__(self):
28 return "Dependency( name=%r, version=%r, relation=%r )" % (self.name, self.version, self.relation )
30 def __eq__(self, other):
31 return self.name == other.name and self.version == other.version and self.relation == other.version
33 def __hash__(self):
34 return hash((self.name, self.version, self.relation))
36 class Package():
37 def __init__(self, filename=None, with_recommends=True):
38 self.name = None
39 self.version = None
40 self.depends_line = None
41 self.filename = None
42 self.with_recommends = with_recommends
44 if filename is not None:
45 self.__init_from_deb_file(os.path.abspath(filename))
47 def __init_from_deb_file(self, filename):
48 assert os.path.isfile(filename)
50 status, output = commands.getstatusoutput("dpkg-deb --info \"%s\"" % filename)
51 if status != 0:
52 print output
53 return
55 for line in output.split("\n"):
56 # remove the leading space
57 line = line[1:]
58 line = line.rstrip()
59 if line[0] == " ":
60 continue
61 bits = line.split(": ", 1)
62 if len(bits) != 2:
63 continue
64 key, value = bits
65 if key == "Depends":
66 self.parse_dependencies( value )
67 if key == "Recommends" and self.with_recommends:
68 self.parse_dependencies( value )
69 if key == "Version":
70 self.version_string = value
71 if key == "Package":
72 self.name = value
76 def __str__(self):
77 return self.name
79 def __repr__(self):
80 return str(self)
82 def parse_dependencies(self, depends_line):
83 depends = []
85 # Assuming that "," is more important than "|"
86 # and that we can only nest them 2 deep at most
87 packages = depends_line.split( "," )
88 for package in packages:
89 package = package.strip()
90 alts = [ s.strip() for s in package.split("|") ]
91 if len(alts) == 1:
92 # just a normal package line
93 depends.append( Dependency( string = package ) )
94 else:
95 depends.append( OrDependencyList( *[ Dependency(string=alt) for alt in alts ] ) )
97 if getattr(self, 'depends', None) is None:
98 # we don't have any depends before
99 self.depends = AndDependencyList( *depends )
100 else:
101 # we have already seen some dependencies, so we need to add these
102 # to that list. This could happen when we are including
103 # recommendations in the dependencies
104 depends.extend(self.depends.dependencies)
105 self.depends = AndDependencyList( *depends )
107 def fulfils(self, dep):
108 """Returns true iff this package can satify the dependency dep"""
109 if dep.name != self.name:
110 # Obviously
111 return False
113 if dep.version is None:
114 # for this dependency the version is unimportant
115 return True
117 # version code from here:
118 # http://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Version
119 # TODO debian group is including the 'ubuntu' string
120 version_re = re.compile("""
121 ((?P<epoch>\d+):)?
123 ( (?P<upstream1>[-0-9.+:]+) - (?P<debian1>[a-zA-Z0-9+.]+) )
124 | ( (?P<upstream2>[0-9.+:]+) ( - (?P<debian2>[a-zA-Z0-9+.]+) )? )
126 """, re.VERBOSE )
128 match = version_re.match( self.version_string )
129 assert match is not None, "The version string for %s (%s) does not match the version regular expression %s" % (self.name, self.version, version_re)
131 dep_match = version_re.match( dep.version )
132 assert dep_match is not None, "The version string for depenency %s (%s) does not match the version regular expression %r" % (dep, dep.version, version_re )
134 relation_to_func = {
135 '<<': operator.lt,
136 '<=': operator.le,
137 '=' : operator.eq,
138 '>=': operator.ge,
139 '>>': operator.gt
142 assert dep.relation in relation_to_func.keys(), "The depenency %s has a relation of %s, which is not in the known relations" % (dep, dep.relation)
143 op = relation_to_func[dep.relation]
145 assert (match.group('epoch') is None and dep_match.group('epoch') is None) or (match.group('epoch') is not None or dep_match.group('epoch') ), "Exactly one of the version strings %s and %s contain an epoch. Either none or both should include one" % (self.version_string, dep.version )
146 if match.group('epoch') is not None:
147 # if we provide an epoch we should check it. If not skip it.
148 if int(match.group('epoch')) != int(dep_match.group('epoch')):
149 return op(int(match.group('epoch')), int(dep_match.group('epoch')))
151 pkg_version_dict = match.groupdict()
152 if pkg_version_dict['upstream1'] is not None:
153 pkg_version_dict['upstream'] = pkg_version_dict['upstream1']
154 pkg_version_dict['debian'] = pkg_version_dict['debian1']
155 else:
156 pkg_version_dict['upstream'] = pkg_version_dict['upstream2']
157 pkg_version_dict['debian'] = pkg_version_dict['debian2']
158 del pkg_version_dict['upstream1'], pkg_version_dict['upstream2']
159 del pkg_version_dict['debian1'], pkg_version_dict['debian2']
161 dep_version_dict = dep_match.groupdict()
162 if dep_version_dict['upstream1'] is not None:
163 dep_version_dict['upstream'] = dep_version_dict['upstream1']
164 dep_version_dict['debian'] = dep_version_dict['debian1']
165 else:
166 dep_version_dict['upstream'] = dep_version_dict['upstream2']
167 dep_version_dict['debian'] = dep_version_dict['debian2']
168 del dep_version_dict['upstream1'], dep_version_dict['upstream2']
169 del dep_version_dict['debian1'], dep_version_dict['debian2']
172 # From: http://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Version
173 # The upstream_version and debian_revision parts are compared by the
174 # package management system using the same algorithm:
176 # The strings are compared from left to right.
178 # First the initial part of each string consisting entirely of
179 # non-digit characters is determined. These two parts (one of which may
180 # be empty) are compared lexically. If a difference is found it is
181 # returned. The lexical comparison is a comparison of ASCII values
182 # modified so that all the letters sort earlier than all the
183 # non-letters.
185 # Then the initial part of the remainder of each string which consists
186 # entirely of digit characters is determined. The numerical values of
187 # these two parts are compared, and any difference found is returned as
188 # the result of the comparison. For these purposes an empty string
189 # (which can only occur at the end of one or both version strings being
190 # compared) counts as zero.
192 # These two steps (comparing and removing initial non-digit strings and
193 # initial digit strings) are repeated until a difference is found or
194 # both strings are exhausted.
198 # Check the upstream version
199 pkg_str = pkg_version_dict['upstream']
200 dep_str = dep_version_dict['upstream']
201 assert pkg_str is not None and dep_str is not None
203 digits_non_digits_pkg = [s for s in re.split("(\D*)(\d*)", pkg_str) if s != ""]
204 digits_non_digits_dep = [s for s in re.split("(\D*)(\d*)", dep_str) if s != ""]
206 smaller_len = min(len(digits_non_digits_dep), len(digits_non_digits_pkg))
208 last_check = None
209 for index in range(smaller_len):
210 try:
211 last_check = op( int(digits_non_digits_pkg[index]), int(digits_non_digits_dep[index]) )
212 except ValueError:
213 last_check = op( digits_non_digits_pkg[index], digits_non_digits_dep[index] )
214 if digits_non_digits_pkg[index] != digits_non_digits_dep[index]:
215 return last_check
217 if pkg_version_dict['debian'] is None or dep_version_dict['debian'] is None:
218 return last_check
220 # Clean up the debian/ubuntu version numbering
221 if re.match( "\d*ubuntu\d*", pkg_version_dict['debian'] ):
222 pkg_version_dict['debian'], pkg_version_dict['ubuntu'] = re.split("ubuntu", pkg_version_dict['debian'])
223 #print repr(pkg_version_dict)
224 if re.match("\d*ubuntu\d*", dep_version_dict['debian'] ):
225 dep_version_dict['debian'], dep_version_dict['ubuntu'] = re.split("ubuntu", dep_version_dict['debian'])
226 #print repr(dep_version_dict)
228 if pkg_version_dict['debian'] != dep_version_dict['debian']:
229 # we need to do the same splitting based on digits and non digits for the debian version.
230 digits_non_digits_pkg = [s for s in re.split("(\D*)(\d*)", pkg_version_dict['debian']) if s != ""]
231 digits_non_digits_dep = [s for s in re.split("(\D*)(\d*)", dep_version_dict['debian']) if s != ""]
233 smaller_len = min(len(digits_non_digits_dep), len(digits_non_digits_pkg))
235 last_check = None
236 for index in range(smaller_len):
237 # check if they are different
238 try:
239 last_check = op( int(digits_non_digits_pkg[index]), int(digits_non_digits_dep[index]) )
240 except ValueError:
241 last_check = op( digits_non_digits_pkg[index], digits_non_digits_dep[index] )
242 if digits_non_digits_pkg[index] != digits_non_digits_dep[index]:
243 return last_check
245 # if we've gotten to here, then the debian versions are the same, so look at the ubuntu version
246 assert pkg_version_dict['debian'] == dep_version_dict['debian'], "When checking debian versions %s and %s, the code got to the 'check ubuntu' version part. This should only happen if the debian versions are the same" % (pkg_version_dict['debian'], dep_version_dict['debian'])
248 # check the ubuntu version
249 if 'ubuntu' not in dep_version_dict.keys() or dep_version_dict['ubuntu'] is None:
250 return last_check
251 else:
252 return op(int(pkg_version_dict['ubuntu']), int(dep_version_dict['ubuntu']))
254 raise NotImplementedError, "The code for comparing versions has finished without returning. This means the author did not fully understand the debian method of comparing versions or this is a funny deb. The deb: %r. The depenency: %r" % (self, dep)
257 def save(self, directory=None):
258 if directory == None:
259 directory = os.getcwd()
261 assert self.filename is not None, "Attempted to save a package with filename = None"
262 assert self.filename[0:7], "Attempted to save a package with a non-http filename. You can't save local packages"
264 print "Downloading "+str(self)
265 urllib.urlretrieve( self.filename, os.path.join( directory, os.path.basename( self.filename ) ) )
268 class DependencyList():
269 def __init__(self, *dependencies):
270 self.dependencies = dependencies
272 def __iter__(self):
273 return self.dependencies.__iter__()
277 class AndDependencyList(DependencyList):
278 def __str__(self):
279 return ", ".join( [ str(dep) for dep in self.dependencies ] )
281 def __repr__(self):
282 return "AndDependencyList( %s )" % ", ".join( [ repr(dep) for dep in self.dependencies ] )
284 class OrDependencyList(DependencyList):
285 def __str__(self):
286 return " | ".join( [ str(dep) for dep in self.dependencies ] )
288 def __repr__(self):
289 return "OrDependencyList( %s )" % ", ".join( [ repr(dep) for dep in self.dependencies ] )
292 class Repository():
293 REMOTE_REPOSITORY, LOCAL_REPOSITORY = range(2)
294 def __init__(self, uri, download_callback_func=None, with_recommends=True):
295 self.packages = []
296 self.type = None
297 self.with_recommends = True
298 if os.path.isdir( os.path.abspath( uri ) ):
299 self.type = Repository.LOCAL_REPOSITORY
300 self.path = os.path.abspath( uri )
301 self.__scan_local_packages()
302 elif uri[0:7] == "http://":
303 self.type = Repository.REMOTE_REPOSITORY
304 self.url = uri
305 self.url, self.dist, self.component = uri.split( " " )
306 self.__scan_remote_packages(download_callback_func=download_callback_func)
308 def __scan_remote_packages(self, download_callback_func=None):
309 # check for the repo file
310 tmpfile_fp, tmpfile = tempfile.mkstemp(suffix=".gz", prefix="web-repo-")
311 urllib.urlretrieve( self.url+ "/dists/" + self.dist + "/" +self.component + "/binary-i386/Packages.gz", filename=tmpfile, reporthook=download_callback_func )
312 self.__scan_packages( gzip.open( tmpfile ) )
315 def __scan_packages(self, releases_fp):
316 package = Package()
317 self.packages = []
318 for line in releases_fp:
319 line = line.rstrip("\n")
320 if line == "":
321 self.packages.append(package)
322 package = Package()
323 continue
324 if line[0] == " ":
325 continue
326 bits = line.split(": ", 1)
327 if len(bits) != 2:
328 continue
329 key, value = bits
330 maps = [ [ 'Package', 'name' ],
331 [ 'Version', 'version_string' ],
332 [ 'Filename', 'filename' ],
334 for key_name, attr in maps:
335 if key == key_name:
336 setattr( package, attr, value )
337 if key == 'Depends':
338 package.parse_dependencies( value )
339 if key == 'Recommends' and self.with_recommends:
340 package.parse_dependencies( value )
341 if key == 'Filename':
342 if self.type == Repository.LOCAL_REPOSITORY:
343 package.filename = os.path.abspath( self.path + "/" + value )
344 elif self.type == Repository.REMOTE_REPOSITORY:
345 package.filename = self.url + "/" + value
349 def __scan_local_packages(self):
350 package = Package()
351 self.packages = []
352 self.__scan_packages( open( self.path + "/binary-i386/Packages" ) )
354 def __contains__(self, package):
355 if isinstance(package, str):
356 # looking for package name
357 pass
358 elif isinstance(package, Package):
359 # looking for an actual package
360 return package in self.dependencies
361 elif isinstance( package, Dependency ):
362 # looking for a version
363 possibilities = [ dep for dep in self.dependencies if deb.name == package.name ]
364 print repr( possibilities )
366 def __getitem__(self, package_name):
367 assert self.packages is not None and len(self.packages) > 0, "The repository %s has no packages, and attempted to get a package from it" % self
368 for pkg in self.packages:
369 if pkg.name == package_name:
370 return pkg
371 return None
373 def __str__(self):
374 print "self.type = " + self.type
375 if self.type == Repository.LOCAL_REPOSITORY:
376 return "\"file://" + os.abspath( self.path ) + "\""
377 elif self.type == Repository.REMOTE_REPOSITORY:
378 return "%s %s %s" % (self.url, self.dist, self.component)
382 def dl_depenencies(packages, local_repos, remote_repos, directory):
383 debs_to_download = set()
384 unmet_dependencies = set()
386 for package in packages:
388 if isinstance(package, str):
389 # deb is a package name
390 options = [r for r in local_repos if r[package] is not None]
391 assert len(options) <= 1, "More than one local repository has the packages %s, don't know what to do" % packages
392 if len(options) == 0:
393 # not found locally, try looking remotely
395 remote_options = [r for r in remote_repos if r[package] is not None]
397 assert len(remote_options) != 0, "package %s is not available in local or remote repositories" % package
398 assert len(remote_options) == 1, "more than one option to download from, there should be only one"
400 package = remote_options[0][package]
401 debs_to_download.add(package)
402 else:
403 package = options[0][package]
406 assert isinstance(package, Package)
408 # Start off with all our depends. We have to keep going until we this
409 # is empty.
410 unfulfiled = set(package.depends)
411 fulfiled_deps = set()
414 while len(unfulfiled) > 0:
415 dependency = unfulfiled.pop()
416 #print "Looking at dependency %s" % dependency
418 fulfiled_locally = False
420 for repo in local_repos:
421 if fulfiled_locally:
422 break
423 for package in repo.packages:
424 if fulfiled_locally:
425 break
426 if isinstance( dependency, OrDependencyList ):
427 if any([package.fulfils(dep) for dep in dependency] ):
428 #print "package %s fulfils the dependency %s, which is part of %s" % (package, dep, dependency)
429 fulfiled_locally = True
430 break
431 else:
432 if package.fulfils(dependency):
433 #print "package %s fulfils the dependency %s" % (package, dependency)
434 fulfiled_locally = True
435 break
437 if fulfiled_locally:
438 # Don't bother looking at the remote repositories if we can already fulfill locally
439 #print "The dependency %s can be fulfilled locally" % dependency
440 continue # to the next package
442 # check if one of the debs we're going to download will fulfil this repository. Can help with cycles
443 if any([deb.fulfils(dependency) for deb in debs_to_download]):
444 #print "The dependency %s will be fulfilled by one of the debs we are going to download" % dependency
445 continue # to next dependency
447 # we know we need to look on the web for this dependency
449 fulfiled_remotely = False
450 remote_repo = None
451 for repo in remote_repos:
452 if fulfiled_remotely:
453 break
454 for package in repo.packages:
455 if fulfiled_remotely:
456 break
457 if isinstance( dependency, OrDependencyList ):
458 if any([package.fulfils(dep) for dep in dependency] ):
459 debs_to_download.add(package)
460 fulfiled_remotely = True
461 remote_repo = repo
462 break
463 else:
464 if package.fulfils(dependency):
465 debs_to_download.add(package)
466 fulfiled_remotely = True
467 remote_repo = repo
468 break
470 #if fulfiled_remotely:
471 #print "The dependency %s can be fulfilled from the %s repository" % (dependency, remote_repo)
473 if not fulfiled_remotely and not fulfiled_locally:
474 print "Warning the dependency %s cannot be fulfilled remotely or locally. Try adding extra repositories" % dependency
475 unmet_dependencies.add(dependency)
478 print "The following packages need to be downloaded: "+", ".join([str(x) for x in debs_to_download])
480 # now download our debs
481 for deb in debs_to_download:
482 deb.save(directory)
484 if len(unmet_dependencies) > 0:
485 print "Warning: the following dependencies could not be met:"
486 print ", ".join([str(x) for x in unmet_dependencies])
487 print "Please add extra repositories"