libs/Project.py

   1 #!/usr/bin/python -tt
   2 # vim: ai ts=4 sts=4 et sw=4
   3
   4 #    Copyright (c) 2007 Intel Corporation
   5 #
   6 #    This program is free software; you can redistribute it and/or modify it
   7 #    under the terms of the GNU General Public License as published by the Free
   8 #    Software Foundation; version 2 of the License
   9 #
  10 #    This program is distributed in the hope that it will be useful, but
  11 #    WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  12 #    or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 #    for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License along
  16 #    with this program; if not, write to the Free Software Foundation, Inc., 59
  17 #    Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18
  19 import errno
  20 import gettext
  21 import glob
  22 import os
  23 import re
  24 import shutil
  25 import socket
  26 import stat
  27 import sys
  28 import time
  29
  30 import mic_cfg
  31 import moblin_pkg
  32 import pdk_utils
  33 import InstallImage
  34 import SDK
  35
  36 debug = False
  37 if mic_cfg.config.has_option('general', 'debug'):
  38     debug = int(mic_cfg.config.get('general', 'debug'))
  39
  40 _ = gettext.lgettext
  41
  42 # This is here for the testing of the new package manager code
  43 USE_NEW_PKG = False
  44 if mic_cfg.config.has_option('general', 'use_new_pkg'):
  45     USE_NEW_PKG = int(mic_cfg.config.get('general', 'use_new_pkg'))
  46
  47 class FileSystem(object):
  48     """
  49     This is the base class for any type of a filesystem.  This is used for both
  50     creating 'jailroot' filesystems that isolate a build from the host Linux
  51     distribution, and also for creating 'target' filesystems that will
  52     eventually be transformed into installation images to be
  53     burned/copied/whatever into the target device.
  54
  55     By just instantiating a FileSystem object, the caller will trigger the
  56     basic root filesystem components to be initialized, but to do anything
  57     usefull with the root filesystem will require the caller to use the
  58     'installPackages' method for installing new RPM packages.
  59     """
  60     def __init__(self, path, progress_callback = None):
  61         if not path:
  62             raise ValueError(_("Empty argument passed in"))
  63         self.progress_callback = progress_callback
  64         self.path = os.path.realpath(os.path.abspath(os.path.expanduser(path)))
  65         self.mounted = []
  66
  67     def updateAndUpgrade(self):
  68         self.mount()
  69         return self.platform.pkg_manager.updateChroot(self.chroot_path,
  70             callback = self.progress_callback)
  71
  72     def setHostname(self, hostname):
  73         self.mount()
  74         # Setup copies of some useful files from the host into the chroot
  75         for filename in ('etc/resolv.conf'),:
  76             source_file = os.path.join(os.sep, filename)
  77             target_file = os.path.join(self.chroot_path, filename)
  78             pdk_utils.safeTextFileCopy(source_file, target_file)
  79         f = open("%s/etc/hostname" % self.path, 'w')
  80         f.write("%s\n" % hostname)
  81         f.close()
  82         f = open("%s/etc/hosts" % self.path, 'w')
  83         f.write("""# Generated by Moblin Image Creator #
  84 127.0.0.1       localhost localhost.localdomain %s
  85
  86 # The following lines are desirable for IPv6 capable hosts
  87 ::1     ip6-localhost ip6-loopback
  88 fe00::0 ip6-localnet
  89 ff00::0 ip6-mcastprefix
  90 ff02::1 ip6-allnodes
  91 ff02::2 ip6-allrouters
  92 ff02::3 ip6-allhosts
  93
  94 """ % hostname)
  95         f.close()
  96
  97     def installPackages(self, packages_list):
  98         self.mount()
  99         return self.platform.pkg_manager.installPackages(self.chroot_path,
 100             packages_list, callback = self.progress_callback)
 101
 102     def chroot(self, cmd, output = None):
 103         if not os.path.isfile(os.path.join(self.chroot_path, 'bin/bash')):
 104             print >> sys.stderr, _("Incomplete jailroot at %s") % (self.chroot_path)
 105             raise ValueError(_("Internal Error: Invalid buildroot at %s") % (self.chroot_path))
 106         self.mount()
 107         if output == None:
 108             output = []
 109         result = pdk_utils.execChrootCommand(self.chroot_path, cmd, output = output, callback = self.progress_callback)
 110         if result != 0:
 111             print _("Error in chroot command exec.  Result: %s") % result
 112             print _("Command was: %s") % cmd
 113             print _("chroot was: %s") % self.chroot_path
 114             sys.stdout.flush()
 115         return result
 116
 117     mount_list = [
 118         # mnt_type, host_dirname, target_dirname, fs_type, device
 119         ('bind', '/tmp', False, None, None),
 120         ('bind', '/usr/share/pdk', False, None, None),
 121         ('host', '/dev/pts', 'dev/pts', 'devpts', 'devpts'),
 122         ('host', '/proc', False, 'proc', 'proc'),
 123         ('host', '/sys', False, 'sysfs', 'sysfs'),
 124     ]
 125
 126     def mount(self):
 127         # We want to keep a list of everything we mount, so that we can use it
 128         # in the umount portion
 129         self.mounted = pdk_utils.mountList(FileSystem.mount_list, self.chroot_path)
 130         self.mounted.extend(self.platform.pkg_manager.mount(self.chroot_path))
 131         # first time mount
 132         buildstamp_path = os.path.join(self.path, 'etc', 'buildstamp')
 133         if not os.path.isfile(buildstamp_path):
 134             buildstamp = open(buildstamp_path, 'w')
 135             print >> buildstamp, "%s %s" % (socket.gethostname(), time.strftime("%d-%m-%Y %H:%M:%S %Z"))
 136             buildstamp.close()
 137
 138     def umount(self, directory_set = None):
 139         """Unmounts the mount points in our target.  On error returns a set
 140         containing the directories that could not be unmounted"""
 141         # Go through all the mount points that we recorded during the mount
 142         # function
 143         if directory_set == None:
 144             directory_set = set()
 145         loop_tries = 8
 146         for x in range(0, loop_tries):
 147             local_set = set()
 148             for mount_point in self.mounted:
 149                 if os.path.exists(mount_point):
 150                     result = pdk_utils.umount(mount_point)
 151                     if not result:
 152                         local_set.add(mount_point)
 153             pdk_utils.umountAllInPath(self.path, local_set)
 154             if not local_set:
 155                 # Success.  All directories could be un-mounted
 156                 break
 157             else:
 158                 for directory in local_set:
 159                     print _("Failed to umount FileSystem directory: %s") % directory
 160                     cmd_line = "lsof | grep %s" % directory
 161                     print _("Execing: %s") % cmd_line
 162                     os.system(cmd_line)
 163                 print _("Sleeping for 5 seconds, try %s of %s") % (x+1, loop_tries)
 164                 time.sleep(5)
 165         directory_set.update(local_set)
 166         return directory_set
 167
 168 class Project(FileSystem):
 169     """
 170     A Project is a type of  'jailroot' filesystem that is used to isolate the
 171     build system from the host Linux distribution.  It also knows how to create
 172     new 'target' filesystems.
 173     """
 174     def __init__(self, config_info, platform, progress_callback = None):
 175         self.path = os.path.realpath(os.path.abspath(os.path.expanduser(config_info.path)))
 176         self.config_info = config_info
 177         self.chroot_path = self.path
 178         self.name = config_info.name
 179         self.platform = platform
 180         self.desc = config_info.desc
 181         self.progress_callback = progress_callback
 182         FileSystem.__init__(self, self.path, progress_callback = progress_callback)
 183
 184         # Create our targets directory
 185         targets_path = os.path.join(self.path, 'targets')
 186         if not os.path.isdir(targets_path):
 187             os.makedirs(targets_path)
 188
 189         # Instantiate all targets
 190         self.targets = {}
 191         for dirname in os.listdir(targets_path):
 192             target = Target(dirname, self, self.progress_callback)
 193             self.targets[target.name] = target
 194
 195     def install(self):
 196         """
 197         Install all the packages defined by Platform.buildroot_packages
 198         """
 199         return super(Project, self).installPackages(self.platform.buildroot_packages + self.platform.buildroot_extras)
 200
 201     def umount(self, directory_set = None):
 202         """Unmount all the directories in our project and any targets in our
 203         project.  On failure will return a set containing directories that
 204         could not be unmounted"""
 205         # We want to umount all of our targets and then anything in our project that we have mounted
 206         if directory_set == None:
 207             directory_set = set()
 208         for target_name in self.targets:
 209             target = self.targets[target_name]
 210             target.umount(directory_set = directory_set)
 211         FileSystem.umount(self, directory_set = directory_set)
 212         if directory_set:
 213             print _("Failed to umount project: %s") % self.path
 214             for directory in directory_set:
 215                 print _("Failed to umount Project directory: %s") % directory
 216                 cmd_line = "lsof | grep %s" % directory
 217                 print _("Execing: %s") % cmd_line
 218                 os.system(cmd_line)
 219         return directory_set
 220
 221     def create_target(self, name, use_rootstrap = True):
 222         if not name:
 223             raise ValueError(_("Target name was not specified"))
 224         if not name in self.targets:
 225             install_path = os.path.join(self.path, 'targets', name, 'fs')
 226             self.platform.createChroot(install_path, use_rootstrap, callback = self.progress_callback)
 227             self.targets[name] = Target(name, self, self.progress_callback)
 228             self.targets[name].mount()
 229             self.targets[name].updateAndUpgrade()
 230             self.targets[name].setHostname('ume')
 231             # Install platform default kernel cmdline
 232             self.set_target_usb_kernel_cmdline(name, self.platform.usb_kernel_cmdline)
 233             self.set_target_hd_kernel_cmdline(name, self.platform.hd_kernel_cmdline)
 234             self.set_target_cd_kernel_cmdline(name, self.platform.cd_kernel_cmdline)
 235         return self.targets[name]
 236
 237     def get_target_usb_kernel_cmdline(self, name):
 238         if not name:
 239            raise ValueError(_("Target name was not specified"))
 240         cmdline = open(os.path.join(self.targets[name].config_path, 'usb_kernel_cmdline'), 'r')
 241         usb_kernel_cmdline = ''
 242         for line in cmdline:
 243             if not re.search(r'^\s*#',line):
 244                 usb_kernel_cmdline += line + ' '
 245         cmdline.close()
 246         return usb_kernel_cmdline.strip()
 247
 248     def get_target_hd_kernel_cmdline(self, name):
 249         if not name:
 250            raise ValueError(_("Target name was not specified"))
 251         cmdline = open(os.path.join(self.targets[name].config_path, 'hd_kernel_cmdline'), 'r')
 252         hd_kernel_cmdline = ''
 253         for line in cmdline:
 254             if not re.search(r'^\s*#',line):
 255                 hd_kernel_cmdline += line + ' '
 256         cmdline.close()
 257         return hd_kernel_cmdline.strip()
 258
 259     def get_target_cd_kernel_cmdline(self, name):
 260         if not name:
 261            raise ValueError(_("Target name was not specified"))
 262         cmdline = open(os.path.join(self.targets[name].config_path, 'cd_kernel_cmdline'), 'r')
 263         cd_kernel_cmdline = ''
 264         for line in cmdline:
 265             if not re.search(r'^\s*#',line):
 266                 cd_kernel_cmdline += line + ' '
 267         cmdline.close()
 268         return cd_kernel_cmdline.strip()
 269
 270     def set_target_usb_kernel_cmdline(self, name, str):
 271         if not name:
 272            raise ValueError(_("Target name was not specified"))
 273         cmdline = open(os.path.join(self.targets[name].config_path, 'usb_kernel_cmdline'), 'w')
 274         print >> cmdline, str
 275         cmdline.close()
 276
 277     def set_target_hd_kernel_cmdline(self, name, str):
 278         if not name:
 279            raise ValueError(_("Target name was not specified"))
 280         cmdline = open(os.path.join(self.targets[name].config_path, 'hd_kernel_cmdline'), 'w')
 281         print >> cmdline, str
 282         cmdline.close()
 283
 284     def set_target_cd_kernel_cmdline(self, name, str):
 285         if not name:
 286            raise ValueError(_("Target name was not specified"))
 287         cmdline = open(os.path.join(self.targets[name].config_path, 'cd_kernel_cmdline'), 'w')
 288         print >> cmdline, str
 289         cmdline.close()
 290
 291     def delete_target(self, name, do_pop=True, callback = None):
 292         target = self.targets[name]
 293         self.umountTarget(target)
 294         seen_paths = []
 295         while True:
 296             try:
 297                 pdk_utils.rmtree(os.path.join(self.path, 'targets', name), callback = callback)
 298                 break
 299             except OSError, e:
 300                 # See if we get a resource busy error, if so we think it is a
 301                 # mounted filesystem issue
 302                 if e.errno == errno.EBUSY:
 303                     if e.filename in seen_paths:
 304                         raise OSError, e
 305                     else:
 306                         seen_paths.append(e.filename)
 307                         pdk_utils.umount(e.filename)
 308                 else:
 309                     raise OSError, e
 310         if do_pop:
 311             self.targets.pop(name)
 312
 313     def umountTarget(self, target):
 314         directory_set = target.umount()
 315         if not directory_set:
 316             # Success, so return
 317             return
 318         # Failed to umount
 319         print _("Failed to umount target: %s") % target.path
 320         cmd_line = "lsof | grep %s" % target.path
 321         print _("Execing: %s") % cmd_line
 322         os.system(cmd_line)
 323         print _("Failed to umount target: %s") % target.path
 324         # Let's remount everything, so stuff will still work
 325         target.mount()
 326         raise pdk_utils.ImageCreatorUmountError, directory_set
 327
 328     def create_live_iso(self, target_name, image_name):
 329         target = self.targets[target_name]
 330         self.umountTarget(target)
 331         image = InstallImage.LiveIsoImage(self, self.targets[target_name], image_name, progress_callback = self.progress_callback)
 332         image.create_image()
 333         target.mount()
 334
 335     def create_install_iso(self, target_name, image_name):
 336         target = self.targets[target_name]
 337         self.umountTarget(target)
 338         image = InstallImage.InstallIsoImage(self, self.targets[target_name], image_name, progress_callback = self.progress_callback)
 339         image.create_image()
 340         target.mount()
 341
 342     def create_live_usb(self, target_name, image_name, type="RAMFS"):
 343         target = self.targets[target_name]
 344         self.umountTarget(target)
 345         image = InstallImage.LiveUsbImage(self, self.targets[target_name], image_name, progress_callback = self.progress_callback)
 346         image.create_image(type)
 347         target.mount()
 348
 349     def create_install_usb(self, target_name, image_name):
 350         target = self.targets[target_name]
 351         self.umountTarget(target)
 352         image = InstallImage.InstallUsbImage(self, self.targets[target_name], image_name, progress_callback = self.progress_callback)
 353         image.create_image()
 354         target.mount()
 355
 356     def tar(self, tar_obj):
 357         """tar up the project.  Need to pass in a tarfile object"""
 358         directory_set = self.umount()
 359         if directory_set:
 360             self.mount()
 361             raise pdk_utils.ImageCreatorUmountError, directory_set
 362         tar_obj.add(self.path, arcname = "project/")
 363
 364     def __str__(self):
 365         return ("<Project: name=%s, path=%s>"
 366                 % (self.name, self.path))
 367     def __repr__(self):
 368         return "Project('%s', '%s', '%s', %s)" % (self.path, self.name, self.desc, self.platform)
 369
 370 class Target(FileSystem):
 371     """
 372     Represents a 'target' filesystem that will eventually be installed on the
 373     target device.
 374     """
 375     def __init__(self, name, project, progress_callback = None):
 376         if not name or not project:
 377             raise ValueError(_("Empty argument passed in"))
 378         self.project = project
 379         self.name = name
 380         self.platform = project.platform
 381         self.top = os.path.join(project.path, "targets", name)
 382
 383         # Load our target's filesystem directory
 384         self.fs_path = os.path.join(self.top, "fs")
 385         self.chroot_path = self.fs_path
 386
 387         # Load/create our target's image directory
 388         self.image_path = os.path.join(self.top, "image")
 389         if not os.path.isdir(self.image_path):
 390             os.makedirs(self.image_path)
 391
 392         # Load/create our target's config directory
 393         self.config_path = os.path.join(self.top, "config")
 394         if not os.path.isdir(self.config_path):
 395             os.makedirs(self.config_path)
 396
 397         # Instantiate the target filesystem
 398         FileSystem.__init__(self, self.fs_path, progress_callback = progress_callback)
 399
 400     def installed_fsets(self):
 401         result = []
 402         for fset in os.listdir(self.top):
 403             if fset not in ['fs', 'image', 'config']:
 404                 result.append(fset)
 405         result.sort()
 406         return result
 407
 408     def __any_fset_installed(self, fset_list):
 409         """See if we have already installed the fset"""
 410         for fset_name in fset_list:
 411             if os.path.isfile(os.path.join(self.top, fset_name)):
 412                 return True
 413         return False
 414
 415     def installFset(self, fset, debug_pkgs = 0, fsets = None, seen_fsets = None):
 416         """
 417         Install a fset into the target filesystem.  If the fsets variable is
 418         supplied with a list of fsets then we will try to recursively install
 419         any missing deps that exist.
 420         """
 421         if os.path.isfile(os.path.join(self.top, fset.name)):
 422             raise ValueError(_("fset %s is already installed!") % (fset.name))
 423
 424         root_fset = False
 425         if not seen_fsets:
 426             print _("Installing Function Set: %s (and any dependencies)") % fset.name
 427             root_fset = True
 428             seen_fsets = set()
 429         if fset.name in seen_fsets:
 430             raise RuntimeError, _("Circular fset dependency encountered for: %s") % fset.name
 431         seen_fsets.add(fset.name)
 432
 433         package_list = []
 434         for dep in fset['deps']:
 435             # An fset "DEP" value can contain a dependency list of the form:
 436             #   DEP=A B|C
 437             # Which means the fset depends on fset A and either fset B or C.
 438             # If B or C are not installed then it will attempt to install the
 439             # first one (fset B).
 440             dep_list = dep.split('|')
 441             if seen_fsets.intersection(dep_list):
 442                 # If any of the fsets we have already seen satisfy the
 443                 # dependency, then continue
 444                 continue
 445             if not self.__any_fset_installed(dep_list):
 446                 if fsets:
 447                     # Determine which fsets are needed to install the required fset
 448                     dependency_list = self.installFset(fsets[dep_list[0]],
 449                      fsets = fsets, debug_pkgs = debug_pkgs, seen_fsets = seen_fsets)
 450                     package_list = package_list + dependency_list
 451                 else:
 452                     raise ValueError(_("fset %s must be installed first!") % (dep_list[0]))
 453         for pkg in fset['pkgs']:
 454             if not pkg in package_list:
 455                 package_list.append(pkg)
 456         if debug_pkgs == 1:
 457             for pkg in fset['debug_pkgs']:
 458                 if not pkg in package_list:
 459                     package_list.append(pkg)
 460         if not root_fset:
 461             return package_list
 462         else:
 463             req_fsets = seen_fsets - set( [fset.name] )
 464             if req_fsets:
 465                 print _("Installing required Function Set: %s") % ' '.join(req_fsets)
 466             self.installPackages(package_list)
 467             # and now create a simple empty file that indicates that the fsets has
 468             # been installed.
 469             for fset_name in seen_fsets:
 470                 fset_file = open(os.path.join(self.top, fset_name), 'w')
 471                 fset_file.close()
 472
 473     def __str__(self):
 474         return ("<Target: name=%s, path=%s, fs_path=%s, image_path=%s, config_path=%s>"
 475                 % (self.name, self.path, self.fs_path, self.image_path, self.config_path))
 476     def __repr__(self):
 477         return "Target('%s', %s)" % (self.path, self.project)
 478
 479 class Callback:
 480     def iteration(process):
 481         return
 482
 483 if __name__ == '__main__':
 484     if len(sys.argv) != 6:
 485         print >> sys.stderr, _("USAGE: %s PROJECT_NAME PROJECT_PATH PROJECT_DESCRIPTION TARGET_NAME PLATFORM_NAME") % (sys.argv[0])
 486         print >> sys.stderr, _("\tPROJECT_NAME: name to call the project.  The config file /usr/share/pdk/projects/project_name.proj is used or created")
 487         print >> sys.stderr, _("\tPROJECT_PATH: directory to install the project")
 488         print >> sys.stderr, _("\tPROJECT_DESCRIPTION: Textual description of the project")
 489         print >> sys.stderr, _("\tTARGET_NAME: ???")
 490         print >> sys.stderr, _("\tPLATFORM_NAME: The platform.  e.g. donley")
 491         sys.exit(1)
 492
 493     name = sys.argv[1]
 494     install_path = os.path.realpath(os.path.abspath(os.path.expanduser(sys.argv[2])))
 495     desc = sys.argv[3]
 496     target_name = sys.argv[4]
 497     platform_name = sys.argv[5]
 498
 499     sdk = SDK.SDK(Callback())
 500
 501     # verify the platform exists
 502     if not platform_name in sdk.platforms:
 503         print >> sys.stderr, _("ERROR: %s is not a valid platform!") % (platform_name)
 504         print >> sys.stderr, _("Available platforms include:")
 505         for key in sorted(sdk.platforms.iterkeys()):
 506             print "\t%s" % (key)
 507         sys.exit(1)
 508     platform = sdk.platforms[platform_name]
 509
 510     # find an existing project, or create a new one
 511     existing_project = False
 512     if name in sdk.projects:
 513         print _("Opening existing project...Using info from config file...")
 514         proj = sdk.projects[name]
 515         existing_project = True
 516     else:
 517         print _("Creating new project...")
 518         proj = sdk.create_project(install_path, name, desc, platform)
 519         proj.install()
 520     print _("Install path: %s") % proj.path
 521     print _("Name: %s") % proj.name
 522     print _("Description: %s") % proj.desc
 523     if existing_project:
 524         print _("Used info from config file: /usr/share/pdk/projects/%s.proj") % name
 525         time.sleep(2)
 526
 527     # see if the target exist
 528     if target_name in proj.targets:
 529         print _("Target already exists: %s") % target_name
 530         print proj.targets
 531     else:
 532         print _("Creating new project target filesystem...")
 533         proj.create_target(target_name)
 534
 535         print _("Installing all available fsets inside target...")
 536         for key in proj.platform.fset:
 537             proj.targets[target_name].installFset(proj.platform.fset[key])
 538