commands/lib/layman/db.py

   1 #!/usr/bin/python
   2 # -*- coding: utf-8 -*-
   3 #################################################################################
   4 # LAYMAN OVERLAY DB
   5 #################################################################################
   6 # File:       db.py
   7 #
   8 #             Access to the db of overlays
   9 #
  10 # Copyright:
  11 #             (c) 2005 - 2008 Gunnar Wrobel
  12 #             Distributed under the terms of the GNU General Public License v2
  13 #
  14 # Author(s):
  15 #             Gunnar Wrobel <wrobel@gentoo.org>
  16 #
  17 '''Handles different storage files.'''
  18
  19 __version__ = "$Id: db.py 309 2007-04-09 16:23:38Z wrobel $"
  20
  21 #===============================================================================
  22 #
  23 # Dependencies
  24 #
  25 #-------------------------------------------------------------------------------
  26
  27 import os, codecs, os.path, urllib2, re, hashlib
  28
  29 from   layman.utils             import path
  30 from   layman.overlay           import Overlays
  31
  32 from   layman.debug             import OUT
  33
  34 #===============================================================================
  35 #
  36 # Class DB
  37 #
  38 #-------------------------------------------------------------------------------
  39
  40 class DB(Overlays):
  41     ''' Handle the list of local overlays.'''
  42
  43     def __init__(self, config):
  44
  45         self.config = config
  46
  47         self.path = config['local_list']
  48
  49         if config['nocheck']:
  50             ignore = 2
  51         else:
  52             ignore = 1
  53
  54         quiet = int(config['quietness']) < 3
  55
  56         Overlays.__init__(self,
  57                           [config['local_list'], ],
  58                           ignore,
  59                           quiet)
  60
  61         OUT.debug('DB handler initiated', 6)
  62
  63     def add(self, overlay, quiet = False):
  64         '''
  65         Add an overlay to the local list of overlays.
  66
  67         >>> write = os.tmpnam()
  68         >>> write2 = os.tmpnam()
  69         >>> write3 = os.tmpnam()
  70         >>> here = os.path.dirname(os.path.realpath(__file__))
  71         >>> config = {'local_list' :
  72         ...           here + '/tests/testfiles/global-overlays.xml',
  73         ...           'make_conf' : write2,
  74         ...           'nocheck'    : True,
  75         ...           'storage'   : write3,
  76         ...           'quietness':3}
  77
  78         >>> here = os.path.dirname(os.path.realpath(__file__))
  79         >>> a = DB(config)
  80         >>> config['local_list'] = write
  81         >>> b = DB(config)
  82         >>> OUT.color_off()
  83
  84         >>> m = MakeConf(config, b.overlays)
  85         >>> m.path = write2
  86         >>> m.write()
  87
  88         Commented out since it needs network access:
  89
  90         # >>> b.add(a.select('wrobel-stable')) #doctest: +ELLIPSIS
  91         # * Running command "/usr/bin/rsync -rlptDvz --progress --delete --delete-after --timeout=180 --exclude="distfiles/*" --exclude="local/*" --exclude="packages/*" "rsync://gunnarwrobel.de/wrobel-stable/*" "/tmp/file.../wrobel-stable""...
  92         # >>> c = Overlays([write, ])
  93         # >>> c.overlays.keys()
  94         # [u'wrobel-stable']
  95
  96         # >>> m = MakeConf(config, b.overlays)
  97         # >>> [i.name for i in m.overlays] #doctest: +ELLIPSIS
  98         # [u'wrobel-stable']
  99
 100
 101         # >>> os.unlink(write)
 102         >>> os.unlink(write2)
 103         >>> import shutil
 104
 105         # >>> shutil.rmtree(write3)
 106         '''
 107
 108         if overlay.name not in self.overlays.keys():
 109             result = overlay.add(self.config['storage'], quiet)
 110             if result == 0:
 111                 if 'priority' in self.config.keys():
 112                     overlay.set_priority(self.config['priority'])
 113                 self.overlays[overlay.name] = overlay
 114                 self.write(self.path)
 115                 make_conf = MakeConf(self.config, self.overlays)
 116                 make_conf.add(overlay)
 117             else:
 118                 raise Exception('Adding the overlay failed! Possible remains of'
 119                                 ' the opration have NOT been removed and may be'
 120                                 ' left at ' + path([self.config['storage'],
 121                                                   overlay.name]) + '. Please re'
 122                                 'move them manually if required.')
 123         else:
 124             raise Exception('Overlay "' + overlay.name + '" already in the loca'
 125                             'l list!')
 126
 127     def delete(self, overlay):
 128         '''
 129         Add an overlay to the local list of overlays.
 130
 131         >>> write = os.tmpnam()
 132         >>> write2 = os.tmpnam()
 133         >>> write3 = os.tmpnam()
 134         >>> here = os.path.dirname(os.path.realpath(__file__))
 135         >>> config = {'local_list' :
 136         ...           here + '/tests/testfiles/global-overlays.xml',
 137         ...           'make_conf' : write2,
 138         ...           'nocheck'    : True,
 139         ...           'storage'   : write3,
 140         ...           'quietness':3}
 141
 142         >>> here = os.path.dirname(os.path.realpath(__file__))
 143         >>> a = DB(config)
 144         >>> config['local_list'] = write
 145         >>> b = DB(config)
 146         >>> OUT.color_off()
 147
 148         >>> m = MakeConf(config, b.overlays)
 149         >>> m.path = here + '/tests/testfiles/make.conf'
 150         >>> m.read()
 151
 152         >>> m.path = write2
 153         >>> m.write()
 154
 155         # >>> b.add(a.select('wrobel-stable')) #doctest: +ELLIPSIS
 156         # * Running command "/usr/bin/rsync -rlptDvz --progress --delete --delete-after --timeout=180 --exclude="distfiles/*" --exclude="local/*" --exclude="packages/*" "rsync://gunnarwrobel.de/wrobel-stable/*" "/tmp/file.../wrobel-stable""...
 157         # >>> b.add(a.select('wrobel')) #doctest: +ELLIPSIS
 158         # * Running command "/usr/bin/svn co "https://overlays.gentoo.org/svn/dev/wrobel/" "/tmp/file.../wrobel""...
 159         # >>> c = Overlays([write, ])
 160         # >>> c.overlays.keys()
 161         # [u'wrobel', u'wrobel-stable']
 162
 163         # >>> b.delete(b.select('wrobel'))
 164         # >>> c = Overlays([write, ])
 165         # >>> c.overlays.keys()
 166         # [u'wrobel-stable']
 167
 168         # >>> m = MakeConf(config, b.overlays)
 169         # >>> [i.name for i in m.overlays] #doctest: +ELLIPSIS
 170         # [u'wrobel-stable']
 171
 172         # >>> os.unlink(write)
 173         >>> os.unlink(write2)
 174         >>> import shutil
 175
 176         # >>> shutil.rmtree(write3)
 177         '''
 178
 179         if overlay.name in self.overlays.keys():
 180             make_conf = MakeConf(self.config, self.overlays)
 181             overlay.delete(self.config['storage'])
 182             del self.overlays[overlay.name]
 183             self.write(self.path)
 184             make_conf.delete(overlay)
 185         else:
 186             raise Exception('No local overlay named "' + overlay.name + '"!')
 187
 188     def sync(self, overlay_name, quiet = False):
 189         '''Synchronize the given overlay.'''
 190
 191         overlay = self.select(overlay_name)
 192
 193         if overlay:
 194             result = overlay.sync(self.config['storage'], quiet)
 195             if result:
 196                 raise Exception('Syncing overlay "' + overlay_name +
 197                                 '" returned status ' + str(result) + '!')
 198         else:
 199             raise Exception('No such overlay ("' + overlay_name + '")!')
 200
 201 #===============================================================================
 202 #
 203 # Class RemoteDB
 204 #
 205 #-------------------------------------------------------------------------------
 206
 207 class RemoteDB(Overlays):
 208     '''Handles fetching the remote overlay list.'''
 209
 210     def __init__(self, config):
 211
 212         self.config = config
 213
 214         self.proxies = {}
 215
 216         if config['proxy']:
 217             self.proxies['http'] = config['proxy']
 218         elif os.getenv('http_proxy'):
 219             self.proxies['http'] = os.getenv('http_proxy')
 220
 221         if self.proxies:
 222             proxy_handler = urllib2.ProxyHandler(self.proxies)
 223             opener = urllib2.build_opener(proxy_handler)
 224             urllib2.install_opener(opener)
 225
 226         self.urls  = [i.strip() for i in config['overlays'].split('\n') if i]
 227
 228         paths = [self.path(i) for i in self.urls]
 229
 230         if config['nocheck']:
 231             ignore = 2
 232         else:
 233             ignore = 0
 234
 235         quiet = int(config['quietness']) < 3
 236
 237         Overlays.__init__(self, paths, ignore, quiet)
 238
 239     def cache(self):
 240         '''
 241         Copy the remote overlay list to the local cache.
 242
 243         >>> here = os.path.dirname(os.path.realpath(__file__))
 244         >>> cache = os.tmpnam()
 245         >>> config = {'overlays' :
 246         ...           'file://' + here + '/tests/testfiles/global-overlays.xml',
 247         ...           'cache' : cache,
 248         ...           'nocheck'    : True,
 249         ...           'proxy' : None,
 250         ...           'quietness':3}
 251         >>> a = RemoteDB(config)
 252         >>> a.cache()
 253         >>> b = open(a.path(config['overlays']))
 254         >>> b.readlines()[24]
 255         '      A collection of ebuilds from Gunnar Wrobel [wrobel@gentoo.org].\\n'
 256
 257         >>> b.close()
 258         >>> os.unlink(a.path(config['overlays']))
 259
 260         >>> a.overlays.keys()
 261         [u'wrobel', u'wrobel-stable']
 262         '''
 263         for url in self.urls:
 264
 265             mpath = self.path(url)
 266
 267             try:
 268
 269                 # Fetch the remote list
 270                 olist = urllib2.urlopen(url).read()
 271
 272                 # Create our storage directory if it is missing
 273                 if not os.path.exists(os.path.dirname(mpath)):
 274                     try:
 275                         os.makedirs(os.path.dirname(mpath))
 276                     except OSError, error:
 277                         raise OSError('Failed to create layman storage direct'
 278                                       + 'ory ' + os.path.dirname(mpath) + '\n'
 279                                       + 'Error was:' + str(error))
 280
 281                 # Before we overwrite the old cache, check that the downloaded
 282                 # file is intact and can be parsed
 283                 try:
 284                     self.read(olist)
 285                 except Exception, error:
 286                     raise IOError('Failed to parse the overlays list fetched fr'
 287                                   'om ' + url + '\nThis means that the download'
 288                                   'ed file is somehow corrupt or there was a pr'
 289                                   'oblem with the webserver. Check the content '
 290                                   'of the file. Error was:\n' + str(error))
 291
 292                 # Ok, now we can overwrite the old cache
 293                 try:
 294                     out_file = open(mpath, 'w')
 295                     out_file.write(olist)
 296                     out_file.close()
 297
 298                 except Exception, error:
 299                     raise IOError('Failed to temporarily cache overlays list in'
 300                                   ' ' + mpath + '\nError was:\n' + str(error))
 301
 302
 303             except IOError, error:
 304                 OUT.warn('Failed to update the overlay list from: '
 305                          + url + '\nError was:\n' + str(error))
 306
 307             try:
 308                 # Finally parse the contents of the cache
 309                 self.read_file(mpath)
 310             except IOError, error:
 311                 OUT.warn('Failed to read a cached version of the overlay list f'
 312                          'rom ' + url + '. You probably did not download the fi'
 313                          'le before. The corresponding entry in your layman.cfg'
 314                          ' file will be disregarded.\nError was:\n' + str(error)
 315                          )
 316
 317     def path(self, url):
 318         '''Return a unique file name for the url.'''
 319
 320         base = self.config['cache']
 321
 322         OUT.debug('Generating cache path.', 6)
 323
 324         return base + '_' + hashlib.md5(url).hexdigest() + '.xml'
 325
 326 #===============================================================================
 327 #
 328 # Helper class MakeConf
 329 #
 330 #-------------------------------------------------------------------------------
 331
 332 class MakeConf:
 333     '''
 334     Handles modifications to /etc/make.conf
 335
 336     Check that an add/remove cycle does not modify the make.conf:
 337
 338     >>> import hashlib
 339     >>> write = os.tmpnam()
 340     >>> here = os.path.dirname(os.path.realpath(__file__))
 341     >>> config = {'local_list' :
 342     ...           here + '/tests/testfiles/global-overlays.xml',
 343     ...           'make_conf' : here + '/tests/testfiles/make.conf',
 344     ...           'nocheck'    : True,
 345     ...           'storage'   : '/usr/portage/local/layman',
 346     ...           'quietness':3}
 347     >>> b = DB(config)
 348     >>> a = MakeConf(config, b.overlays)
 349     >>> o_md5 = str(hashlib.md5(open(here + '/tests/testfiles/make.conf').read()).hexdigest())
 350     >>> a.path = write
 351     >>> a.add(b.overlays['wrobel-stable'])
 352     >>> [i.name for i in a.overlays]
 353     [u'wrobel-stable', u'wrobel-stable']
 354     >>> a.add(b.overlays['wrobel'])
 355     >>> [i.name for i in a.overlays]
 356     [u'wrobel', u'wrobel-stable', u'wrobel-stable']
 357     >>> a.delete(b.overlays['wrobel-stable'])
 358     >>> [i.name for i in a.overlays]
 359     [u'wrobel']
 360     >>> a.add(b.overlays['wrobel-stable'])
 361     >>> [i.name for i in a.overlays]
 362     [u'wrobel', u'wrobel-stable']
 363     >>> a.delete(b.overlays['wrobel'])
 364     >>> n_md5 = str(hashlib.md5(open(write).read()).hexdigest())
 365     >>> o_md5 == n_md5
 366     True
 367     >>> os.unlink(write)
 368     '''
 369
 370     my_re = re.compile('PORTDIR_OVERLAY\s*=\s*"([^"]*)"')
 371
 372     def __init__(self, config, overlays):
 373
 374         self.path = config['make_conf']
 375         self.storage = config['storage']
 376         self.data = ''
 377         self.db = overlays
 378         self.overlays = []
 379         self.extra = []
 380
 381         self.read()
 382
 383     def add(self, overlay):
 384         '''
 385         Add an overlay to make.conf.
 386
 387         >>> write = os.tmpnam()
 388         >>> here = os.path.dirname(os.path.realpath(__file__))
 389         >>> config = {'local_list' :
 390         ...           here + '/tests/testfiles/global-overlays.xml',
 391         ...           'make_conf' : here + '/tests/testfiles/make.conf',
 392         ...           'nocheck'    : True,
 393         ...           'storage'   : '/usr/portage/local/layman',
 394         ...           'quietness':3}
 395         >>> c = DB(config)
 396         >>> a = MakeConf(config, c.overlays)
 397         >>> a.path = write
 398         >>> a.add(c.select('wrobel'))
 399         >>> config['make_conf'] = write
 400         >>> b = MakeConf(config, c.overlays)
 401         >>> [i.name for i in b.overlays]
 402         [u'wrobel', u'wrobel-stable']
 403         >>> b.extra
 404         [u'/usr/portage/local/ebuilds/testing', u'/usr/portage/local/ebuilds/stable', u'/usr/portage/local/kolab2', u'/usr/portage/local/gentoo-webapps-overlay/experimental', u'/usr/portage/local/gentoo-webapps-overlay/production-ready']
 405
 406         >>> os.unlink(write)
 407         '''
 408         self.overlays.append(overlay)
 409         self.write()
 410
 411     def delete(self, overlay):
 412         '''
 413         Delete an overlay from make.conf.
 414
 415         >>> write = os.tmpnam()
 416         >>> here = os.path.dirname(os.path.realpath(__file__))
 417         >>> config = {'local_list' :
 418         ...           here + '/tests/testfiles/global-overlays.xml',
 419         ...           'make_conf' : here + '/tests/testfiles/make.conf',
 420         ...           'nocheck'    : True,
 421         ...           'storage'   : '/usr/portage/local/layman',
 422         ...           'quietness':3}
 423         >>> c = DB(config)
 424         >>> a = MakeConf(config, c.overlays)
 425         >>> a.path = write
 426         >>> a.delete(c.select('wrobel-stable'))
 427         >>> config['make_conf'] = write
 428         >>> b = MakeConf(config, c.overlays)
 429         >>> [i.name for i in b.overlays]
 430         []
 431         >>> b.extra
 432         [u'/usr/portage/local/ebuilds/testing', u'/usr/portage/local/ebuilds/stable', u'/usr/portage/local/kolab2', u'/usr/portage/local/gentoo-webapps-overlay/experimental', u'/usr/portage/local/gentoo-webapps-overlay/production-ready']
 433
 434         >>> os.unlink(write)
 435         '''
 436         self.overlays = [i
 437                          for i in self.overlays
 438                          if i.name != overlay.name]
 439         self.write()
 440
 441     def read(self):
 442         '''
 443         Read the list of registered overlays from /etc/make.conf.
 444
 445         >>> here = os.path.dirname(os.path.realpath(__file__))
 446         >>> config = {'local_list' :
 447         ...           here + '/tests/testfiles/global-overlays.xml',
 448         ...           'make_conf' : here + '/tests/testfiles/make.conf',
 449         ...           'nocheck'    : True,
 450         ...           'storage'   : '/usr/portage/local/layman',
 451         ...           'quietness':3}
 452         >>> c = DB(config)
 453         >>> a = MakeConf(config, c.overlays)
 454         >>> [i.name for i in a.overlays]
 455         [u'wrobel-stable']
 456         >>> a.extra
 457         [u'/usr/portage/local/ebuilds/testing', u'/usr/portage/local/ebuilds/stable', u'/usr/portage/local/kolab2', u'/usr/portage/local/gentoo-webapps-overlay/experimental', u'/usr/portage/local/gentoo-webapps-overlay/production-ready']
 458         '''
 459         if os.path.isfile(self.path):
 460             self.content()
 461
 462             overlays = self.my_re.search(self.data)
 463
 464             if not overlays:
 465                 raise Exception('Did not find a PORTDIR_OVERLAY entry in file ' +
 466                                 self.path +'! Did you specify the correct file?')
 467
 468             overlays = [i.strip()
 469                         for i in overlays.group(1).split('\n')
 470                         if i.strip()]
 471
 472             for i in overlays:
 473                 if i[:len(self.storage)] == self.storage:
 474                     oname = os.path.basename(i)
 475                     if  oname in self.db.keys():
 476                         self.overlays.append(self.db[oname])
 477                     else:
 478                         # These are additional overlays that we dont know
 479                         # anything about. The user probably added them manually
 480                         self.extra.append(i)
 481                 else:
 482                     # These are additional overlays that we dont know anything
 483                     # about. The user probably added them manually
 484                     self.extra.append(i)
 485
 486
 487         else:
 488             self.overlays = []
 489             self.data     = 'PORTDIR_OVERLAY="\n"\n'
 490
 491         self.extra = [i for i in self.extra
 492                          if (i != '$PORTDIR_OVERLAY'
 493                              and i != '${PORTDIR_OVERLAY}')]
 494
 495     def write(self):
 496         '''
 497         Write the list of registered overlays to /etc/make.conf.
 498
 499         >>> write = os.tmpnam()
 500         >>> here = os.path.dirname(os.path.realpath(__file__))
 501         >>> config = {'local_list' :
 502         ...           here + '/tests/testfiles/global-overlays.xml',
 503         ...           'make_conf' : here + '/tests/testfiles/make.conf',
 504         ...           'nocheck'    : True,
 505         ...           'storage'   : '/usr/portage/local/layman',
 506         ...           'quietness':3}
 507         >>> c = DB(config)
 508         >>> a = MakeConf(config, c.overlays)
 509         >>> a.path = write
 510         >>> a.write()
 511         >>> config['make_conf'] = write
 512         >>> b = MakeConf(config, c.overlays)
 513         >>> [i.name for i in b.overlays]
 514         [u'wrobel-stable']
 515         >>> b.extra
 516         [u'/usr/portage/local/ebuilds/testing', u'/usr/portage/local/ebuilds/stable', u'/usr/portage/local/kolab2', u'/usr/portage/local/gentoo-webapps-overlay/experimental', u'/usr/portage/local/gentoo-webapps-overlay/production-ready']
 517
 518         >>> os.unlink(write)
 519         '''
 520         def prio_sort(a, b):
 521             '''Sort by priority.'''
 522             if a.priority < b.priority:
 523                 return -1
 524             elif a.priority > b.priority:
 525                 return 1
 526             return 0
 527
 528         self.overlays.sort(prio_sort)
 529
 530         paths = []
 531         for i in self.overlays:
 532             paths.append(path((self.storage, i.name, )))
 533
 534         overlays = 'PORTDIR_OVERLAY="\n'
 535         overlays += '\n'.join(paths) + '\n'
 536         overlays += '$PORTDIR_OVERLAY\n'
 537         overlays += '\n'.join(self.extra)
 538         overlays += '"'
 539
 540         content = self.my_re.sub(overlays, self.data)
 541
 542         if not self.my_re.search(content):
 543             raise Exception('Ups, failed to set a proper PORTDIR_OVERLAY entry '
 544                             'in file ' + self.path +'! Did not overwrite the fi'
 545                             'le.')
 546
 547         try:
 548             make_conf = codecs.open(self.path, 'w', 'utf-8')
 549
 550             make_conf.write(content)
 551
 552             make_conf.close()
 553
 554         except Exception, error:
 555             raise Exception('Failed to read "' + self.path + '".\nError was:\n'
 556                             + str(error))
 557
 558     def content(self):
 559         '''
 560         Returns the content of the /etc/make.conf file.
 561         '''
 562         try:
 563             make_conf = codecs.open(self.path, 'r', 'utf-8')
 564
 565             self.data = make_conf.read()
 566
 567             make_conf.close()
 568
 569         except Exception, error:
 570             raise Exception('Failed to read "' + self.path + '".\nError was:\n'
 571                             + str(error))
 572
 573 #===============================================================================
 574 #
 575 # Testing
 576 #
 577 #-------------------------------------------------------------------------------
 578
 579 if __name__ == '__main__':
 580     import doctest, sys
 581
 582     # Ignore warnings here. We are just testing
 583     from warnings     import filterwarnings, resetwarnings
 584     filterwarnings('ignore')
 585
 586     doctest.testmod(sys.modules[__name__])
 587
 588     resetwarnings()