libs/pdk_utils.py

   1 #!/usr/bin/python -tt
   2 # vim: ai ts=4 sts=4 et sw=4
   3
   4 #    Copyright (c) 2007 Intel Corporation
   5 #
   6 #    This program is free software; you can redistribute it and/or modify it
   7 #    under the terms of the GNU General Public License as published by the Free
   8 #    Software Foundation; version 2 of the License
   9 #
  10 #    This program is distributed in the hope that it will be useful, but
  11 #    WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  12 #    or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 #    for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License along
  16 #    with this program; if not, write to the Free Software Foundation, Inc., 59
  17 #    Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18
  19 # This file contains utility functions which do not need to be inside any of
  20 # our current classes
  21
  22 import exceptions
  23 import fcntl
  24 import gettext
  25 import gobject
  26 import os
  27 import re
  28 import select
  29 import stat
  30 import subprocess
  31 import sys
  32 import tempfile
  33 import shutil
  34 import time
  35
  36 import mic_cfg
  37
  38 _ = gettext.lgettext
  39
  40 # this is for the copySourcesListFile() function
  41 src_regex = None
  42 CONFIG_DIR = os.path.expanduser("~/.image-creator")
  43 if not os.path.isdir(CONFIG_DIR):
  44     print _("~/.image-creator/ directory did not exist.  Creating")
  45     os.makedirs(CONFIG_DIR)
  46
  47 sources_regex_file = os.path.expanduser(os.path.join(CONFIG_DIR, "sources_cfg"))
  48 if os.path.isfile(sources_regex_file):
  49     f = open(sources_regex_file, "r")
  50     mirrorSelectionLine = f.readline()
  51     f.close()
  52     global_dict = {}
  53     execfile(sources_regex_file, global_dict)
  54     if mirrorSelectionLine.find("=") != -1:
  55         mirrorToUse = mirrorSelectionLine.split('=')[1]
  56         mirrorToUse = mirrorToUse[1:-2]
  57         if mirrorToUse != "no_mirror":
  58             if mirrorToUse in global_dict:
  59                 src_regex = global_dict[mirrorToUse]
  60             else:
  61                 if 'sources_regex' in global_dict:
  62                     src_regex = global_dict['sources_regex']
  63     else:
  64         if 'sources_regex' in global_dict:
  65             src_regex = global_dict['sources_regex']
  66 else:
  67     print _("Creating sample %s file") % sources_regex_file
  68     out_file = open(sources_regex_file, 'w')
  69     print  >> out_file, """#!/usr/bin/python
  70
  71 # If you have a local mirror of the Ubuntu and/or Moblin.org APT repositories,
  72 # then this configuration file will be useful to you.
  73
  74 # This file is used when copying the files that will go into
  75 # /etc/apt/sources.list.d/  It consists of a list, which contains a search
  76 # regular expression and a replacement string.  When copying the files into the
  77 # /etc/apt/sources.list.d/ , of the projects and targets, a search and replace
  78 # will be performed.
  79
  80 sources_regex = [
  81     # source_archive,                           local mirror of source archive
  82
  83 # Edit the following and uncomment them to enable use of a local mirror server.
  84 # NOTE: The trailing space is important in the strings!!!!
  85 #    (r'http://archive.ubuntu.com/ubuntu ',       'http://<PATH_TO_YOUR_LOCAL_MIRROR_OF_ARCHIVES_UBUNTU_COM/ '),
  86 #    (r'http://ports.ubuntu.com/ubuntu-ports ', 'http://<PATH_TO_YOUR_LOCAL_MIRROR_OF_PORTS_UBUNTU_COM/ '),
  87 #    (r'http://www.moblin.org/apt ',       'http://<PATH_TO_YOUR_LOCAL_MIRROR_OF_MOBLIN_ORG/ '),
  88
  89 ]"""
  90     out_file.close()
  91
  92 # Make sure the file and directory are owned by the user if it is currently
  93 # owned by root.
  94 userid = int(mic_cfg.config.get("userinfo", "userid"))
  95 groupid = int(mic_cfg.config.get("userinfo", "groupid"))
  96 for filename in [ CONFIG_DIR, sources_regex_file ]:
  97     stat_val = os.stat(filename)
  98     st_uid = stat_val[stat.ST_UID]
  99     st_gid = stat_val[stat.ST_GID]
 100     do_chmod = False
 101     if st_uid == 0 and userid != 0:
 102         do_chmod = True
 103     elif st_gid == 0 and groupid != 0:
 104         do_chmod = True
 105     if do_chmod:
 106         os.chown(filename, userid, groupid)
 107
 108 def main():
 109     # Add something to exercise this code
 110     print _("USB devices: %s") % get_current_udisks()
 111
 112 def areWeRoot():
 113     """Figure out if we are running as root"""
 114     sudo = int(mic_cfg.config.get('userinfo', 'sudo'))
 115     userid = int(mic_cfg.config.get('userinfo', 'userid'))
 116     if sudo == 0 and userid != 0:
 117         return False
 118     return True
 119
 120 def get_current_udisks():
 121     usb_devices = []
 122     dirname = os.path.realpath(os.path.abspath('/sys/bus/scsi'))
 123     work_list = getUsbDirTree(dirname)
 124     usb_list = [ x for x in work_list if re.search(r'usb', x) ]
 125     for filename in usb_list:
 126         device_dir = os.path.join('/sys/devices', filename)
 127         if os.path.isdir(device_dir):
 128             for device_file in os.listdir(device_dir):
 129                 full_path = os.path.join(device_dir, device_file)
 130                 result = re.search(r'^block:(?P<dev>.*)', device_file)
 131                 if result:
 132                     usb_dev = os.path.join('/dev', result.group('dev'))
 133                     if os.path.exists(usb_dev):
 134                         usb_devices.append(usb_dev)
 135     return usb_devices
 136
 137 def getUsbDirTree(dirname):
 138     file_set = set()
 139     for filename in os.listdir(dirname):
 140         full_path = os.path.join(dirname, filename)
 141         if os.path.islink(full_path):
 142             file_set.add(os.path.realpath(full_path))
 143         elif os.path.isdir(full_path):
 144             file_set.update(getUsbDirTree(full_path))
 145         else:
 146             file_set.add(full_path)
 147     return file_set
 148
 149 def umountAllInPath(dirname, directory_set = None):
 150     """Unmount all mounts that are found underneath the dirname specified.  On
 151     error returns a set containing the list of directories that failed to
 152     unmount"""
 153     # Have to add a '/' on the end to prevent /foo/egg and /foo/egg2 being
 154     # treated as if both were under /foo/egg
 155     if directory_set == None:
 156         directory_set = set()
 157     our_path = os.path.realpath(os.path.abspath(dirname)) + os.sep
 158     mounts = getMountInfo()
 159     for mpoint in mounts:
 160         fs_type=mounts[mpoint].fs_type
 161         mpoint = os.path.realpath(os.path.abspath(mpoint)) + os.sep
 162         if our_path == mpoint[:len(our_path)]:
 163             # need save tmpfs to 'real' dir
 164             if fs_type == 'tmpfs':
 165                 tmp_path = tempfile.mkdtemp('','pdk-', '/tmp')
 166                 os.rmdir(tmp_path)
 167                 shutil.copytree(mpoint, tmp_path)
 168             result = umount(mpoint)
 169             if not result:
 170                 directory_set.add(os.path.abspath(mpoint))
 171                 if fs_type == 'tmpfs':
 172                     shutil.rmtree(tmp_path)
 173             elif fs_type == 'tmpfs':
 174                 os.rmdir(mpoint)
 175                 shutil.move(tmp_path, mpoint)
 176     return directory_set
 177
 178 def umount(dirname):
 179     """Helper function to un-mount a directory, returns back False if it failed
 180     to unmount the directory"""
 181     dirname = os.path.abspath(os.path.expanduser(dirname))
 182     result = os.system("umount %s" % (dirname))
 183     if result:
 184         # umount failed, see if the directory is actually mounted, if it isn't
 185         # then we think it is okay
 186         mounts = getMountInfo()
 187         if dirname in mounts:
 188             return False
 189     return True
 190
 191 def umount_device(device_file):
 192     """umount a device if it is mounted"""
 193     search_file = "%s " % os.path.realpath(os.path.abspath(device_file))
 194     mount_file = open('/proc/mounts', 'r')
 195     for line in mount_file:
 196         line = line.strip()
 197         if line.find(search_file) == 0:
 198             print _("Umounting: %s") % device_file
 199             return umount(device_file)
 200     return True
 201
 202 def getMountInfo():
 203     """Function to parse the list of mounts and return back the data"""
 204     output = {}
 205     mounts_file = "/proc/mounts"
 206     in_file = open(mounts_file, 'r')
 207     for line in in_file:
 208         line = line.strip()
 209         mount_info = MountInfo(line)
 210         output[mount_info.dirname] = mount_info
 211     return output
 212
 213 def ismount(path):
 214     """Function to see if a path is mounted, this is because os.path.ismount()
 215     does not seem to detect --bind"""
 216     path = os.path.realpath(os.path.abspath(os.path.expanduser(path)))
 217     output = []
 218     cmd = "mount"
 219     result = execCommand(cmd, quiet = True, output = output)
 220     for line in output:
 221         result = re.search(r'(?P<dev>.*) on (?P<mnt_point>.*) type', line)
 222         if result:
 223             mnt_point = result.group('mnt_point')
 224             if mnt_point == path:
 225                 return True
 226     return False
 227
 228
 229 def setblocking(f, flag):
 230     " set/clear blocking mode"
 231     # get the file descriptor
 232     fd = f.fileno()
 233     # get the file's current flag settings
 234     fl = fcntl.fcntl(fd, fcntl.F_GETFL)
 235     if flag:
 236         # clear non-blocking mode from flags
 237         fl = fl & ~os.O_NONBLOCK
 238     else:
 239         # set non-blocking mode from flags
 240         fl = fl | os.O_NONBLOCK
 241     # update the file's flags
 242     fcntl.fcntl(fd, fcntl.F_SETFL, fl)
 243
 244 def execCommand(cmd_line, quiet = False, output = None, callback = None):
 245         if output == None and callback == None:
 246             p = subprocess.Popen(cmd_line.split())
 247         else:
 248             p = subprocess.Popen(cmd_line.split(), stdout = subprocess.PIPE, stderr = subprocess.STDOUT, stdin = subprocess.PIPE, close_fds = True)
 249         # To get the callbacks to work, we need to capture the output
 250         if callback != None and output == None:
 251             output = []
 252         # Don't ever want the process waiting on stdin.
 253         if output != None:
 254             p.stdin.close()
 255             # Make the stdout of the subprocess non-blocking, so that we won't
 256             # ever hang waiting for it.  This way our callback function should
 257             # always keep being called.
 258             setblocking(p.stdout, False)
 259         # This is so that we can keep calling our callback
 260         poll = select.poll()
 261         if output != None:
 262             poll.register(p.stdout, select.POLLIN)
 263         out_buffer = ""
 264         # As long as our subprocess returns None, then the subprocess is still
 265         # running.
 266         while p.poll() == None:
 267             if output != None:
 268                 # Only do this if we are capturing output
 269                 result = poll.poll(10)
 270                 if result:
 271                     buf = p.stdout.read()
 272                     if buf != "":
 273                         out_buffer += buf
 274                         if not quiet:
 275                             sys.stdout.write(buf)
 276             if callback:
 277                 callback(p)
 278         if output != None:
 279             # Have to scan for anything remaining in the subprocess stdout
 280             # buffer
 281             while True:
 282                 buf = p.stdout.read()
 283                 if buf == "":
 284                     break
 285                 out_buffer += buf
 286                 if not quiet:
 287                     sys.stdout.write(buf)
 288             # Now we have to package up our output
 289             for line in out_buffer.splitlines():
 290                 output.append(line)
 291         result = p.returncode
 292         return result
 293
 294 def execChrootCommand(path, cmd, output = None, callback = None):
 295     if not os.path.isfile(os.path.join(path, 'bin/bash')):
 296         print >> sys.stderr, _("Incomplete jailroot at %s") % (path)
 297         raise ValueError(_("Internal Error: Invalid buildroot at %s") % (path))
 298     if output == None:
 299         output = []
 300     cmd_line = "/usr/sbin/chroot %s %s" % (path, cmd)
 301     result = execCommand(cmd_line, output = output, callback = callback)
 302     if result != 0:
 303         print _("Error in chroot.  Result: %s") % result
 304         print _("Command was: %s") % cmd_line
 305         sys.stdout.flush()
 306     return result
 307
 308 def copySourcesListFile(sourcefile, destfile):
 309     """The purpose of this function is allow the user to be able to point at a
 310     local repository for some of the sources, rather than going out over the
 311     Internet"""
 312     in_file = open(sourcefile, 'r')
 313     out_file = open(destfile, 'w')
 314     for line in in_file:
 315         line = line.strip()
 316         if type(src_regex) == type([]):
 317             for regex, sub in src_regex:
 318                 line = re.sub(regex, sub, line)
 319         print >> out_file, line
 320     in_file.close()
 321     out_file.close()
 322
 323 def touchFile(filename):
 324     """Work like the 'touch' command"""
 325     if not os.path.exists(filename):
 326         basedir = os.path.dirname(filename)
 327         if not os.path.exists(basedir):
 328             os.makedirs(basedir)
 329         out_file = open(filename, 'w')
 330         out_file.close()
 331     os.utime(filename, None)
 332
 333 def rmtree(path, ignore_errors=False, onerror=None, callback = None, count = 0):
 334     """Recursively delete a directory tree.
 335
 336     This function was copied from the shutil.py library in Python 2.5.1.  The
 337     license for Python is GPL compatible so it is okay to use it.  The reason
 338     for creating a custom version of this is so that we can have a callback
 339     function.  This way our throbber won't die for long periods of time when
 340     deleting a large directory tree.
 341
 342     If ignore_errors is set, errors are ignored; otherwise, if onerror
 343     is set, it is called to handle the error with arguments (func,
 344     path, exc_info) where func is os.listdir, os.remove, or os.rmdir;
 345     path is the argument to that function that caused it to fail; and
 346     exc_info is a tuple returned by sys.exc_info().  If ignore_errors
 347     is false and onerror is None, an exception is raised.
 348     """
 349     if ignore_errors:
 350         def onerror(*args):
 351             pass
 352     elif onerror is None:
 353         def onerror(*args):
 354             raise
 355     names = []
 356     try:
 357         names = os.listdir(path)
 358     except os.error, err:
 359         onerror(os.listdir, path, sys.exc_info())
 360     for name in names:
 361         count += 1
 362         # For every 200 files deleted, lets call our callback
 363         if count % 200 == 0 and callback:
 364             callback(None)
 365         fullname = os.path.join(path, name)
 366         try:
 367             mode = os.lstat(fullname).st_mode
 368         except os.error:
 369             mode = 0
 370         if stat.S_ISDIR(mode):
 371             rmtree(fullname, ignore_errors, onerror, callback = callback, count = count)
 372         else:
 373             try:
 374                 os.remove(fullname)
 375             except os.error, err:
 376                 onerror(os.remove, fullname, sys.exc_info())
 377     try:
 378         os.rmdir(path)
 379     except os.error:
 380         onerror(os.rmdir, path, sys.exc_info())
 381
 382 def copy(src, dst, callback = None):
 383     if callback:
 384         timer = gobject.timeout_add(100, callback, None)
 385         pid = os.fork()
 386         if (pid == 0):
 387             # child process
 388             shutil.copy(src, dst)
 389             os._exit(0)
 390         while 1:
 391             pid, exit_status = os.waitpid(pid, os.WNOHANG)
 392             if pid == 0:
 393                 callback(None)
 394                 time.sleep(0.1)
 395             else:
 396                 gobject.source_remove(timer)
 397                 break
 398             if exit_status:
 399                 # Something failed, try again without the fork
 400                 shutil.copy(src, dst)
 401     else:
 402         shutil.copy(src, dst)
 403
 404 def safeTextFileCopy(source_file, destination_file, force = False):
 405     """Routine which attempts to safely copy a text file.  This means that if
 406     we have the destination file and it has our signature text in it, then we
 407     will overwrite it.  But if the signature text is not in the destination
 408     file then we will not overwrite it"""
 409     id_string = _("# ##-Created by Moblin Image Creator: if this line exists we will overwrite this file -##")
 410     copyfile = False
 411     if os.path.isfile(source_file):
 412         if not os.path.isfile(destination_file) or force:
 413             copyfile = True
 414         else:
 415             in_file = open(destination_file, 'r')
 416             for line in in_file:
 417                 line = line.strip()
 418                 if re.search(r'^' + id_string, line):
 419                     copyfile = True
 420                     break
 421             in_file.close()
 422     if copyfile:
 423         in_file = open(source_file, 'r')
 424         out_file = open(destination_file, 'w')
 425         print >> out_file, id_string
 426         for line in in_file:
 427             out_file.write(line)
 428         in_file.close()
 429         out_file.close()
 430
 431 def mountList(mount_list, chroot_dir):
 432     """Mount the items specified in the mount list.  Return back a list of what
 433     got mounted"""
 434     # We want to keep a list of everything we mount, so that we can use it in
 435     # the umount portion
 436     mounted_list = []
 437     mounts = getMountInfo()
 438     for mnt_type, host_dirname, target_dirname, fs_type, device in mount_list:
 439         # If didn't specify target_dirname then use the host_dirname path,
 440         # but we have to remove the leading '/'
 441         if not target_dirname and host_dirname:
 442             target_dirname = re.sub(r'^' + os.sep, '', host_dirname)
 443         # Do the --bind mount types
 444         if mnt_type == "bind":
 445             path = os.path.join(chroot_dir, target_dirname)
 446             mounted_list.append(path)
 447             if not os.path.isdir(path):
 448                 os.makedirs(path)
 449             if not ismount(path) and os.path.isdir(host_dirname):
 450                 result = os.system('mount --bind %s %s' % (host_dirname, path))
 451                 if result != 0:
 452                     raise OSError(_("Internal error while attempting to bind mount /%s!") % (host_dirname))
 453         # Mimic host mounts, if possible
 454         elif mnt_type == 'host':
 455             if host_dirname in mounts:
 456                 mount_info = mounts[host_dirname]
 457                 fs_type = mount_info.fs_type
 458                 device = mount_info.device
 459                 options = "-o %s" % mount_info.options
 460             else:
 461                 options = ""
 462             path = os.path.join(chroot_dir, target_dirname)
 463             mounted_list.append(path)
 464             if not os.path.isdir(path):
 465                 os.makedirs(path)
 466             if not ismount(path):
 467                 cmd = 'mount %s -t %s %s %s' % (options, fs_type, device, path)
 468                 result = execCommand(cmd)
 469                 if result != 0:
 470                     raise OSError(_("Internal error while attempting to mount %s %s!") % (host_dirname, target_dirname))
 471     return mounted_list
 472
 473 class MountInfo(object):
 474     def __init__(self, mount_line):
 475         """Input is in the form that is found in /etc/mtab (or /proc/mounts)"""
 476         mount_line = mount_line.strip()
 477         result = mount_line.split()
 478         self.device = result[0]
 479         self.dirname = result[1]
 480         self.fs_type = result[2]
 481         self.options = result[3]
 482     def __str__(self):
 483         return ("%s %s %s %s" % (self.device, self.dirname, self.fs_type, self.options))
 484     def __repr__(self):
 485         return "MountInfo('%s')" % self.__str__()
 486
 487 # An exception class for Image Creator
 488 class ImageCreatorError(exceptions.Exception):
 489     def __init__(self, args=None):
 490         self.args = args
 491
 492
 493 class ImageCreatorUmountError(ImageCreatorError):
 494     def __init__(self, directory_set = set()):
 495         self.directory_set = directory_set
 496
 497 def getAllProcesses():
 498     """Return back a dictionary of all the currently running processes on the
 499     system"""
 500     processes = {}
 501     dirname = "/proc"
 502     for filename in sorted(os.listdir(dirname)):
 503         full_path = os.path.join(dirname, filename)
 504         if os.path.isdir(full_path) and re.search(r'^[0-9]+$', filename):
 505             pid = int(filename)
 506             in_file = open(os.path.join(full_path, "status"))
 507             for line in in_file:
 508                 line = line.strip()
 509                 result = re.search(r'^ppid:\t(?P<ppid>.*)', line, re.I)
 510                 if result:
 511                     ppid = int(result.group('ppid'))
 512                     processes.setdefault(ppid, []).append(pid)
 513                     break
 514             in_file.close()
 515     return processes
 516
 517 def findChildren(pid, process_dict = None):
 518     """Find all the children of PID (Process ID).  Does not return back PID"""
 519     output = []
 520     if not process_dict:
 521         process_dict = getAllProcesses()
 522     if pid in process_dict:
 523         for child_pid in process_dict[pid]:
 524             output.append(child_pid)
 525             output.extend(findChildren(child_pid, process_dict))
 526     output.sort()
 527     return output
 528
 529 def signalChildren(pid, send_signal, process_dict = None, parent_first = True):
 530     """Send the signal 'send_signal' to the parent and all of its children
 531         parent_first = Send the signal to the parent before signaling the
 532         children"""
 533     if not process_dict:
 534         process_dict = getAllProcesses()
 535     if pid in process_dict:
 536         if parent_first:
 537             print _("Sending signal: %s to PID: %s") % (send_signal, pid)
 538             try:
 539                 os.kill(pid, send_signal)
 540             except:
 541                 pass
 542         for child_pid in process_dict[pid]:
 543             signalChildren(child_pid, send_signal = send_signal,
 544                 process_dict = process_dict, parent_first = parent_first)
 545         if not parent_first:
 546             print _("Sending signal: %s to PID: %s") % (send_signal, pid)
 547             try:
 548                 os.kill(pid, send_signal)
 549             except:
 550                 pass
 551
 552 if '__main__' == __name__:
 553     sys.exit(main())