2 # -*- coding: utf-8 -*-
3 #################################################################################
5 #################################################################################
8 # Access to the db of overlays
11 # (c) 2005 - 2008 Gunnar Wrobel
12 # Distributed under the terms of the GNU General Public License v2
15 # Gunnar Wrobel <wrobel@gentoo.org>
17 '''Handles different storage files.'''
19 __version__
= "$Id: db.py 309 2007-04-09 16:23:38Z wrobel $"
21 #===============================================================================
25 #-------------------------------------------------------------------------------
27 import os
, codecs
, os
.path
, urllib2
, re
, hashlib
29 from layman
.utils
import path
30 from layman
.overlay
import Overlays
32 from layman
.debug
import OUT
34 #===============================================================================
38 #-------------------------------------------------------------------------------
41 ''' Handle the list of local overlays.'''
43 def __init__(self
, config
):
47 self
.path
= config
['local_list']
54 quiet
= int(config
['quietness']) < 3
56 Overlays
.__init
__(self
,
57 [config
['local_list'], ],
61 OUT
.debug('DB handler initiated', 6)
63 def add(self
, overlay
, quiet
= False):
65 Add an overlay to the local list of overlays.
67 >>> write = os.tmpnam()
68 >>> write2 = os.tmpnam()
69 >>> write3 = os.tmpnam()
70 >>> here = os.path.dirname(os.path.realpath(__file__))
71 >>> config = {'local_list' :
72 ... here + '/tests/testfiles/global-overlays.xml',
73 ... 'make_conf' : write2,
75 ... 'storage' : write3,
78 >>> here = os.path.dirname(os.path.realpath(__file__))
80 >>> config['local_list'] = write
84 >>> m = MakeConf(config, b.overlays)
88 Commented out since it needs network access:
90 # >>> b.add(a.select('wrobel-stable')) #doctest: +ELLIPSIS
91 # * Running command "/usr/bin/rsync -rlptDvz --progress --delete --delete-after --timeout=180 --exclude="distfiles/*" --exclude="local/*" --exclude="packages/*" "rsync://gunnarwrobel.de/wrobel-stable/*" "/tmp/file.../wrobel-stable""...
92 # >>> c = Overlays([write, ])
93 # >>> c.overlays.keys()
96 # >>> m = MakeConf(config, b.overlays)
97 # >>> [i.name for i in m.overlays] #doctest: +ELLIPSIS
101 # >>> os.unlink(write)
102 >>> os.unlink(write2)
105 # >>> shutil.rmtree(write3)
108 if overlay
.name
not in self
.overlays
.keys():
109 result
= overlay
.add(self
.config
['storage'], quiet
)
111 if 'priority' in self
.config
.keys():
112 overlay
.set_priority(self
.config
['priority'])
113 self
.overlays
[overlay
.name
] = overlay
114 self
.write(self
.path
)
115 make_conf
= MakeConf(self
.config
, self
.overlays
)
116 make_conf
.add(overlay
)
118 raise Exception('Adding the overlay failed! Possible remains of'
119 ' the opration have NOT been removed and may be'
120 ' left at ' + path([self
.config
['storage'],
121 overlay
.name
]) + '. Please re'
122 'move them manually if required.')
124 raise Exception('Overlay "' + overlay
.name
+ '" already in the loca'
127 def delete(self
, overlay
):
129 Add an overlay to the local list of overlays.
131 >>> write = os.tmpnam()
132 >>> write2 = os.tmpnam()
133 >>> write3 = os.tmpnam()
134 >>> here = os.path.dirname(os.path.realpath(__file__))
135 >>> config = {'local_list' :
136 ... here + '/tests/testfiles/global-overlays.xml',
137 ... 'make_conf' : write2,
138 ... 'nocheck' : True,
139 ... 'storage' : write3,
142 >>> here = os.path.dirname(os.path.realpath(__file__))
144 >>> config['local_list'] = write
148 >>> m = MakeConf(config, b.overlays)
149 >>> m.path = here + '/tests/testfiles/make.conf'
155 # >>> b.add(a.select('wrobel-stable')) #doctest: +ELLIPSIS
156 # * Running command "/usr/bin/rsync -rlptDvz --progress --delete --delete-after --timeout=180 --exclude="distfiles/*" --exclude="local/*" --exclude="packages/*" "rsync://gunnarwrobel.de/wrobel-stable/*" "/tmp/file.../wrobel-stable""...
157 # >>> b.add(a.select('wrobel')) #doctest: +ELLIPSIS
158 # * Running command "/usr/bin/svn co "https://overlays.gentoo.org/svn/dev/wrobel/" "/tmp/file.../wrobel""...
159 # >>> c = Overlays([write, ])
160 # >>> c.overlays.keys()
161 # [u'wrobel', u'wrobel-stable']
163 # >>> b.delete(b.select('wrobel'))
164 # >>> c = Overlays([write, ])
165 # >>> c.overlays.keys()
168 # >>> m = MakeConf(config, b.overlays)
169 # >>> [i.name for i in m.overlays] #doctest: +ELLIPSIS
172 # >>> os.unlink(write)
173 >>> os.unlink(write2)
176 # >>> shutil.rmtree(write3)
179 if overlay
.name
in self
.overlays
.keys():
180 make_conf
= MakeConf(self
.config
, self
.overlays
)
181 overlay
.delete(self
.config
['storage'])
182 del self
.overlays
[overlay
.name
]
183 self
.write(self
.path
)
184 make_conf
.delete(overlay
)
186 raise Exception('No local overlay named "' + overlay
.name
+ '"!')
188 def sync(self
, overlay_name
, quiet
= False):
189 '''Synchronize the given overlay.'''
191 overlay
= self
.select(overlay_name
)
194 result
= overlay
.sync(self
.config
['storage'], quiet
)
196 raise Exception('Syncing overlay "' + overlay_name
+
197 '" returned status ' + str(result
) + '!')
199 raise Exception('No such overlay ("' + overlay_name
+ '")!')
201 #===============================================================================
205 #-------------------------------------------------------------------------------
207 class RemoteDB(Overlays
):
208 '''Handles fetching the remote overlay list.'''
210 def __init__(self
, config
):
217 self
.proxies
['http'] = config
['proxy']
218 elif os
.getenv('http_proxy'):
219 self
.proxies
['http'] = os
.getenv('http_proxy')
222 proxy_handler
= urllib2
.ProxyHandler(self
.proxies
)
223 opener
= urllib2
.build_opener(proxy_handler
)
224 urllib2
.install_opener(opener
)
226 self
.urls
= [i
.strip() for i
in config
['overlays'].split('\n') if i
]
228 paths
= [self
.path(i
) for i
in self
.urls
]
230 if config
['nocheck']:
235 quiet
= int(config
['quietness']) < 3
237 Overlays
.__init
__(self
, paths
, ignore
, quiet
)
241 Copy the remote overlay list to the local cache.
243 >>> here = os.path.dirname(os.path.realpath(__file__))
244 >>> cache = os.tmpnam()
245 >>> config = {'overlays' :
246 ... 'file://' + here + '/tests/testfiles/global-overlays.xml',
248 ... 'nocheck' : True,
251 >>> a = RemoteDB(config)
253 >>> b = open(a.path(config['overlays']))
254 >>> b.readlines()[24]
255 ' A collection of ebuilds from Gunnar Wrobel [wrobel@gentoo.org].\\n'
258 >>> os.unlink(a.path(config['overlays']))
260 >>> a.overlays.keys()
261 [u'wrobel', u'wrobel-stable']
263 for url
in self
.urls
:
265 mpath
= self
.path(url
)
269 # Fetch the remote list
270 olist
= urllib2
.urlopen(url
).read()
272 # Create our storage directory if it is missing
273 if not os
.path
.exists(os
.path
.dirname(mpath
)):
275 os
.makedirs(os
.path
.dirname(mpath
))
276 except OSError, error
:
277 raise OSError('Failed to create layman storage direct'
278 + 'ory ' + os
.path
.dirname(mpath
) + '\n'
279 + 'Error was:' + str(error
))
281 # Before we overwrite the old cache, check that the downloaded
282 # file is intact and can be parsed
285 except Exception, error
:
286 raise IOError('Failed to parse the overlays list fetched fr'
287 'om ' + url
+ '\nThis means that the download'
288 'ed file is somehow corrupt or there was a pr'
289 'oblem with the webserver. Check the content '
290 'of the file. Error was:\n' + str(error
))
292 # Ok, now we can overwrite the old cache
294 out_file
= open(mpath
, 'w')
295 out_file
.write(olist
)
298 except Exception, error
:
299 raise IOError('Failed to temporarily cache overlays list in'
300 ' ' + mpath
+ '\nError was:\n' + str(error
))
303 except IOError, error
:
304 OUT
.warn('Failed to update the overlay list from: '
305 + url
+ '\nError was:\n' + str(error
))
308 # Finally parse the contents of the cache
309 self
.read_file(mpath
)
310 except IOError, error
:
311 OUT
.warn('Failed to read a cached version of the overlay list f'
312 'rom ' + url
+ '. You probably did not download the fi'
313 'le before. The corresponding entry in your layman.cfg'
314 ' file will be disregarded.\nError was:\n' + str(error
)
318 '''Return a unique file name for the url.'''
320 base
= self
.config
['cache']
322 OUT
.debug('Generating cache path.', 6)
324 return base
+ '_' + hashlib
.md5(url
).hexdigest() + '.xml'
326 #===============================================================================
328 # Helper class MakeConf
330 #-------------------------------------------------------------------------------
334 Handles modifications to /etc/make.conf
336 Check that an add/remove cycle does not modify the make.conf:
339 >>> write = os.tmpnam()
340 >>> here = os.path.dirname(os.path.realpath(__file__))
341 >>> config = {'local_list' :
342 ... here + '/tests/testfiles/global-overlays.xml',
343 ... 'make_conf' : here + '/tests/testfiles/make.conf',
344 ... 'nocheck' : True,
345 ... 'storage' : '/usr/portage/local/layman',
348 >>> a = MakeConf(config, b.overlays)
349 >>> o_md5 = str(hashlib.md5(open(here + '/tests/testfiles/make.conf').read()).hexdigest())
351 >>> a.add(b.overlays['wrobel-stable'])
352 >>> [i.name for i in a.overlays]
353 [u'wrobel-stable', u'wrobel-stable']
354 >>> a.add(b.overlays['wrobel'])
355 >>> [i.name for i in a.overlays]
356 [u'wrobel', u'wrobel-stable', u'wrobel-stable']
357 >>> a.delete(b.overlays['wrobel-stable'])
358 >>> [i.name for i in a.overlays]
360 >>> a.add(b.overlays['wrobel-stable'])
361 >>> [i.name for i in a.overlays]
362 [u'wrobel', u'wrobel-stable']
363 >>> a.delete(b.overlays['wrobel'])
364 >>> n_md5 = str(hashlib.md5(open(write).read()).hexdigest())
370 my_re
= re
.compile('PORTDIR_OVERLAY\s*=\s*"([^"]*)"')
372 def __init__(self
, config
, overlays
):
374 self
.path
= config
['make_conf']
375 self
.storage
= config
['storage']
383 def add(self
, overlay
):
385 Add an overlay to make.conf.
387 >>> write = os.tmpnam()
388 >>> here = os.path.dirname(os.path.realpath(__file__))
389 >>> config = {'local_list' :
390 ... here + '/tests/testfiles/global-overlays.xml',
391 ... 'make_conf' : here + '/tests/testfiles/make.conf',
392 ... 'nocheck' : True,
393 ... 'storage' : '/usr/portage/local/layman',
396 >>> a = MakeConf(config, c.overlays)
398 >>> a.add(c.select('wrobel'))
399 >>> config['make_conf'] = write
400 >>> b = MakeConf(config, c.overlays)
401 >>> [i.name for i in b.overlays]
402 [u'wrobel', u'wrobel-stable']
404 [u'/usr/portage/local/ebuilds/testing', u'/usr/portage/local/ebuilds/stable', u'/usr/portage/local/kolab2', u'/usr/portage/local/gentoo-webapps-overlay/experimental', u'/usr/portage/local/gentoo-webapps-overlay/production-ready']
408 self
.overlays
.append(overlay
)
411 def delete(self
, overlay
):
413 Delete an overlay from make.conf.
415 >>> write = os.tmpnam()
416 >>> here = os.path.dirname(os.path.realpath(__file__))
417 >>> config = {'local_list' :
418 ... here + '/tests/testfiles/global-overlays.xml',
419 ... 'make_conf' : here + '/tests/testfiles/make.conf',
420 ... 'nocheck' : True,
421 ... 'storage' : '/usr/portage/local/layman',
424 >>> a = MakeConf(config, c.overlays)
426 >>> a.delete(c.select('wrobel-stable'))
427 >>> config['make_conf'] = write
428 >>> b = MakeConf(config, c.overlays)
429 >>> [i.name for i in b.overlays]
432 [u'/usr/portage/local/ebuilds/testing', u'/usr/portage/local/ebuilds/stable', u'/usr/portage/local/kolab2', u'/usr/portage/local/gentoo-webapps-overlay/experimental', u'/usr/portage/local/gentoo-webapps-overlay/production-ready']
437 for i
in self
.overlays
438 if i
.name
!= overlay
.name
]
443 Read the list of registered overlays from /etc/make.conf.
445 >>> here = os.path.dirname(os.path.realpath(__file__))
446 >>> config = {'local_list' :
447 ... here + '/tests/testfiles/global-overlays.xml',
448 ... 'make_conf' : here + '/tests/testfiles/make.conf',
449 ... 'nocheck' : True,
450 ... 'storage' : '/usr/portage/local/layman',
453 >>> a = MakeConf(config, c.overlays)
454 >>> [i.name for i in a.overlays]
457 [u'/usr/portage/local/ebuilds/testing', u'/usr/portage/local/ebuilds/stable', u'/usr/portage/local/kolab2', u'/usr/portage/local/gentoo-webapps-overlay/experimental', u'/usr/portage/local/gentoo-webapps-overlay/production-ready']
459 if os
.path
.isfile(self
.path
):
462 overlays
= self
.my_re
.search(self
.data
)
465 raise Exception('Did not find a PORTDIR_OVERLAY entry in file ' +
466 self
.path
+'! Did you specify the correct file?')
468 overlays
= [i
.strip()
469 for i
in overlays
.group(1).split('\n')
473 if i
[:len(self
.storage
)] == self
.storage
:
474 oname
= os
.path
.basename(i
)
475 if oname
in self
.db
.keys():
476 self
.overlays
.append(self
.db
[oname
])
478 # These are additional overlays that we dont know
479 # anything about. The user probably added them manually
482 # These are additional overlays that we dont know anything
483 # about. The user probably added them manually
489 self
.data
= 'PORTDIR_OVERLAY="\n"\n'
491 self
.extra
= [i
for i
in self
.extra
492 if (i
!= '$PORTDIR_OVERLAY'
493 and i
!= '${PORTDIR_OVERLAY}')]
497 Write the list of registered overlays to /etc/make.conf.
499 >>> write = os.tmpnam()
500 >>> here = os.path.dirname(os.path.realpath(__file__))
501 >>> config = {'local_list' :
502 ... here + '/tests/testfiles/global-overlays.xml',
503 ... 'make_conf' : here + '/tests/testfiles/make.conf',
504 ... 'nocheck' : True,
505 ... 'storage' : '/usr/portage/local/layman',
508 >>> a = MakeConf(config, c.overlays)
511 >>> config['make_conf'] = write
512 >>> b = MakeConf(config, c.overlays)
513 >>> [i.name for i in b.overlays]
516 [u'/usr/portage/local/ebuilds/testing', u'/usr/portage/local/ebuilds/stable', u'/usr/portage/local/kolab2', u'/usr/portage/local/gentoo-webapps-overlay/experimental', u'/usr/portage/local/gentoo-webapps-overlay/production-ready']
521 '''Sort by priority.'''
522 if a
.priority
< b
.priority
:
524 elif a
.priority
> b
.priority
:
528 self
.overlays
.sort(prio_sort
)
531 for i
in self
.overlays
:
532 paths
.append(path((self
.storage
, i
.name
, )))
534 overlays
= 'PORTDIR_OVERLAY="\n'
535 overlays
+= '\n'.join(paths
) + '\n'
536 overlays
+= '$PORTDIR_OVERLAY\n'
537 overlays
+= '\n'.join(self
.extra
)
540 content
= self
.my_re
.sub(overlays
, self
.data
)
542 if not self
.my_re
.search(content
):
543 raise Exception('Ups, failed to set a proper PORTDIR_OVERLAY entry '
544 'in file ' + self
.path
+'! Did not overwrite the fi'
548 make_conf
= codecs
.open(self
.path
, 'w', 'utf-8')
550 make_conf
.write(content
)
554 except Exception, error
:
555 raise Exception('Failed to read "' + self
.path
+ '".\nError was:\n'
560 Returns the content of the /etc/make.conf file.
563 make_conf
= codecs
.open(self
.path
, 'r', 'utf-8')
565 self
.data
= make_conf
.read()
569 except Exception, error
:
570 raise Exception('Failed to read "' + self
.path
+ '".\nError was:\n'
573 #===============================================================================
577 #-------------------------------------------------------------------------------
579 if __name__
== '__main__':
582 # Ignore warnings here. We are just testing
583 from warnings
import filterwarnings
, resetwarnings
584 filterwarnings('ignore')
586 doctest
.testmod(sys
.modules
[__name__
])