bin/add-debs-to-cd-image.py

   1 #! /usr/bin/python
   2
   3 from optparse import OptionParser
   4 import os, shutil, tempfile, commands, glob, sys, re, subprocess, shutil, time
   5
   6 usage = "%prog [options] <list of .debs>"
   7
   8 parser = OptionParser(usage=usage)
   9
  10 parser.add_option("-d", "--cd-dir",
  11                   dest="cddir", default=None,
  12                   help="The directory of the install cd details")
  13
  14 parser.add_option( "-k", "--gpg-key",
  15                     dest="gpgkey", default=None, type="string",
  16                     help="The GPG key used to sign the packages" )
  17
  18 parser.add_option( "--ubuntu-keyring",
  19                    dest="keyring", default=None, type="string",
  20                    help="The directory of the ubuntu keyring source. if not provided it will be downloaded." )
  21
  22 parser.add_option( "--indices",
  23                    dest="indices", default=None, type="string",
  24                    help="The directory of the ubuntu indices" )
  25
  26 parser.add_option( "--passphrase",
  27                    dest="passphrase", default=None, type="string",
  28                    help="The passphrase for the gpg key" )
  29
  30 parser.add_option( "--dist",
  31                    dest="dist", default=None, type="string",
  32                    help="Instead of autodetecting the dist, use this" )
  33
  34 (options, orig_debs) = parser.parse_args()
  35
  36 if not orig_debs or not options.cddir or not options.gpgkey or not options.indices:
  37     parser.print_help()
  38     sys.exit(1)
  39
  40 assert os.path.isdir( options.indices )
  41
  42 debs = [os.path.abspath( deb ) for deb in orig_debs ]
  43
  44 cddir = os.path.abspath( options.cddir )
  45 indices = os.path.abspath( options.indices )
  46
  47 if options.dist is None:
  48     # see what the name of the distro is, (eg dapper, edgy.. )
  49     dists_dir = [ dir for dir in os.listdir( os.path.join ( cddir, 'dists' ) ) if dir not in ['stable', 'unstable' ] ]
  50
  51     assert len(dists_dir) == 1
  52     dist = dists_dir[0]
  53 else:
  54     dist = options.dist
  55
  56 dist_name_to_version = { 'warty':'4.10', 'hoary':'5.04', 'breezy':'5.10',
  57                          'dapper':'6.06', 'edgy':'6.10', 'feisty':'7.04',
  58                          'gutsy':'7.10', 'hardy':'8.04', 'intrepid':'8.10' }
  59
  60 assert dist in dist_name_to_version.keys()
  61
  62 def RunCommand(cmd, msg=None):
  63     print "running : %s" % cmd
  64     p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  65     returncode = p.wait()
  66     print "output : ", p.stdout.read()
  67     print "stderr: ", p.stderr.read()
  68     if returncode != 0:
  69       print "FAILED!"
  70       if msg is not None: print msg
  71       sys.exit(returncode)
  72
  73 def makedir_if_not_exist( *path_parts ):
  74     path = os.path.join( *path_parts )
  75     if not os.path.exists( path ):
  76         os.makedirs( path )
  77
  78 makedir_if_not_exist( cddir, 'dists', dist, 'extras', 'binary-i386' )
  79 makedir_if_not_exist( cddir, 'pool', 'extras' )
  80 makedir_if_not_exist( cddir, 'isolinux' )
  81 makedir_if_not_exist( cddir, 'preseed' )
  82
  83 # Copy all the debs to the extras file
  84 for deb in debs:
  85     print "Copying %s" % deb
  86     shutil.copy( deb, os.path.join( cddir, "pool", "extras" ) )
  87
  88 releases_file = open( os.path.join( cddir, 'dists', dist, 'extras', 'binary-i386', 'Release' ), 'w' )
  89 releases_file.write( "Archive: " + dist + "\n" )
  90 releases_file.write( "Version: " + dist_name_to_version[dist] + "\n" )
  91 releases_file.write( "Component: extras\nOrigin: Ubuntu\nLabel: Ubuntu\nArchitecture: i386\n")
  92 releases_file.close()
  93 print "Wrote the Releases file"
  94
  95
  96 old_cwd = os.getcwd()
  97 temp_dir = tempfile.mkdtemp( prefix="/tmp/camarabuntu-tmp-", dir=old_cwd)
  98 os.chdir( temp_dir )
  99
 100 #shutil.copytree( os.path.join( old_cwd, cddir, 'pool', 'main', 'u', 'ubuntu-keyring' ), temp_dir )
 101 #assert false;
 102
 103
 104 # TODO different versions of ubuntu-keyring
 105 if options.keyring is None:
 106     # download a new one
 107     print "Downloading the ubuntu-keyring..."
 108     cmd =  "apt-get source ubuntu-keyring"
 109     msg = """An error occured when downloading the source of the ubuntu-keyring package
 110     This is needed to sign our new packages"""
 111     RunCommand(cmd, msg=msg)
 112 else:
 113     if options.keyring[-1] == '/':
 114         # This should be a directory, if there's a / at the end, it'll break os.path.split
 115         options.keyring = options.keyring[0:-1]
 116     shutil.copytree( os.path.join( old_cwd, options.keyring ), os.path.join( temp_dir, os.path.split( options.keyring )[1] ) )
 117
 118 # find the file
 119 gpg_keys_filename = glob.glob ( os.path.join( temp_dir, 'ubuntu-keyring*/keyrings/ubuntu-archive-keyring.gpg' ) )[0]
 120 ubuntu_keyring_dir = [name for name in glob.glob( os.path.join( temp_dir, 'ubuntu-keyring*' ) ) if os.path.isdir(name)][0]
 121
 122 assert os.path.isfile( gpg_keys_filename )
 123
 124 print "Adding GPG key %s to the ubuntu-keyring" % options.gpgkey
 125
 126 cmd = "gpg --import < %s" % gpg_keys_filename
 127 # if this fails someone was messing with the code before
 128 RunCommand(cmd)
 129
 130 cmd = "gpg --export FBB75451 437D05B5 %s > %s" % (options.gpgkey, gpg_keys_filename)
 131 # Invalid keys are not detected here, unfortunatly
 132 RunCommand(cmd)
 133
 134 print "\n\nRebuilding the ubuntu-keyring."
 135 os.chdir( ubuntu_keyring_dir )
 136 # Invalid keys are not detected here, unfortunatly
 137 cmd = "dpkg-buildpackage -rfakeroot -k%s" % options.gpgkey
 138 msg = """Unable to rebuild the ubuntu-keyring
 139     Possible causes:
 140     (*) The GPG key you gave (%s) is invalid, check available keys with "gpg --list-keys" """ % options.gpgkey
 141 RunCommand(cmd, msg=msg)
 142 print "Finished Rebuilding the ubuntu-keyring.\n\n"
 143
 144 # Copy these files into the main component
 145 shutil.copy( glob.glob( os.path.join( temp_dir, "ubuntu-keyring*_all.deb" ) )[0], os.path.join( old_cwd, cddir, 'pool/main/u/ubuntu-keyring/' ) )
 146
 147
 148 # Clean up, remove all this crap
 149 os.chdir( old_cwd )
 150 shutil.rmtree( temp_dir )
 151
 152
 153 ## Now create the indices.
 154 temp_dir = tempfile.mkdtemp( prefix="/tmp/camarabuntu-tmp-", dir=old_cwd )
 155 os.chdir( temp_dir )
 156
 157 ftparchive_deb = open( os.path.join( temp_dir, 'apt-ftparchive-deb.conf' ), 'w' )
 158 ftparchive_deb.write( """Dir {
 159   ArchiveDir "%(cddir)s";
 160 };
 161
 162 TreeDefault {
 163   Directory "pool/";
 164 };
 165
 166 BinDirectory "pool/main" {
 167   Packages "dists/%(dist)s/main/binary-i386/Packages";
 168   BinOverride "%(indices)s/override.%(dist)s.main";
 169   ExtraOverride "%(indices)s/override.%(dist)s.extra.main";
 170 };
 171
 172 BinDirectory "pool/restricted" {
 173  Packages "dists/%(dist)s/restricted/binary-i386/Packages";
 174  BinOverride "%(indices)s/override.%(dist)s.restricted";
 175 };
 176
 177 Default {
 178   Packages {
 179     Extensions ".deb";
 180     Compress ". gzip";
 181   };
 182 };
 183
 184 Contents {
 185   Compress "gzip";
 186 };
 187 """ % {'cddir':cddir, 'dist':dist, 'indices':indices} )
 188
 189 ftparchive_deb.close()
 190
 191 # generate the extras override. This is converted from the perl version on
 192 # https://help.ubuntu.com/community/InstallCDCustomization
 193
 194 main_packages = open( os.path.join( old_cwd, cddir, 'dists', dist, 'main/binary-i386/Packages' ) )
 195 extra_overrides = open( os.path.join( temp_dir, "override.%s.extra.main" % dist ), 'w' )
 196 task = None
 197 for line in main_packages:
 198     line = line.rstrip("\n")
 199     line = line.rstrip("\r\n")
 200     line = line.rstrip("\r")
 201     if re.search("^\s", line):
 202         continue # to the next line
 203     if line == "" and task is not None:
 204         extra_overrides.write("%s Task %s\n" % (package, task))
 205         package = None
 206         task = None
 207     if line == "":
 208         continue
 209     key, value = line.split(": ", 1)
 210     if key == "Package":
 211         package = value
 212     if key == "Task":
 213         task = value
 214
 215
 216 extra_overrides.close()
 217 main_packages.close()
 218
 219 ftparchive_udeb = open( os.path.join( temp_dir, "apt-ftparchive-udeb.conf" ), 'w')
 220 ftparchive_udeb.write("""Dir {
 221   ArchiveDir "%(cddir)s";
 222 };
 223
 224 TreeDefault {
 225   Directory "pool/";
 226 };
 227
 228 BinDirectory "pool/main" {
 229   Packages "dists/%(dist)s/main/debian-installer/binary-i386/Packages";
 230   BinOverride "%(indices)s/override.%(dist)s.main.debian-installer";
 231 };
 232
 233 BinDirectory "pool/restricted" {
 234   Packages "dists/%(dist)s/restricted/debian-installer/binary-i386/Packages";
 235   BinOverride "%(indices)s/override.%(dist)s.restricted.debian-installer";
 236 };
 237
 238 Default {
 239   Packages {
 240     Extensions ".udeb";
 241     Compress ". gzip";
 242   };
 243 };
 244
 245 Contents {
 246   Compress "gzip";
 247 };
 248 """ % {'cddir':cddir, 'dist':dist, 'indices':indices} )
 249 ftparchive_udeb.close()
 250
 251 ftparchive_extras = open( os.path.join( temp_dir, 'apt-ftparchive-extras.conf' ), 'w')
 252 ftparchive_extras.write("""Dir {
 253   ArchiveDir "%(cddir)s";
 254 };
 255
 256 TreeDefault {
 257   Directory "pool/";
 258 };
 259
 260 BinDirectory "pool/extras" {
 261   Packages "dists/%(dist)s/extras/binary-i386/Packages";
 262 };
 263
 264 Default {
 265   Packages {
 266     Extensions ".deb";
 267     Compress ". gzip";
 268   };
 269 };
 270
 271 Contents {
 272   Compress "gzip";
 273 };
 274 """ % {'cddir':cddir, 'dist':dist, 'indices':indices} )
 275 ftparchive_extras.close()
 276
 277 release_file = open( os.path.join( temp_dir, "release.conf" ), 'w' )
 278 release_file.write("""
 279 APT::FTPArchive::Release::Origin "Ubuntu";
 280 APT::FTPArchive::Release::Label "Ubuntu";
 281 APT::FTPArchive::Release::Suite "%(dist)s";
 282 APT::FTPArchive::Release::Version "%(dist_number)s";
 283 APT::FTPArchive::Release::Codename "%(dist)s";
 284 APT::FTPArchive::Release::Architectures "i386";
 285 APT::FTPArchive::Release::Components "main restricted extras";
 286 APT::FTPArchive::Release::Description "Ubuntu %(dist_number)s";
 287 """ % {'cddir':cddir, 'dist':dist, 'indices':indices, 'dist_number':dist_name_to_version[dist]} )
 288 release_file.close()
 289
 290 # now build the repository
 291
 292 print "Generating the APT repository..."
 293
 294 status, output = commands.getstatusoutput("apt-ftparchive -c %(temp_dir)s/release.conf generate %(temp_dir)s/apt-ftparchive-deb.conf" % {'temp_dir':temp_dir})
 295 if status != 0:
 296     print output
 297 assert status == 0
 298
 299 status, output = commands.getstatusoutput("apt-ftparchive -c %(temp_dir)s/release.conf generate %(temp_dir)s/apt-ftparchive-udeb.conf" % {'temp_dir':temp_dir})
 300 if status != 0:
 301     print output
 302 assert status == 0
 303
 304 status, output = commands.getstatusoutput("apt-ftparchive -c %(temp_dir)s/release.conf generate %(temp_dir)s/apt-ftparchive-extras.conf"% {'temp_dir':temp_dir})
 305 if status != 0:
 306     print output
 307 assert status == 0
 308
 309 if os.path.isfile( os.path.join( cddir, 'dists', dist, 'Release.gpg') ):
 310     # Otherwise apt-ftparchive will ask to overwrite it, we want this whole process automated
 311     os.remove( os.path.join( cddir, 'dists', dist, 'Release.gpg' ) )
 312 status, output = commands.getstatusoutput("apt-ftparchive -c %(temp_dir)s/release.conf release %(cddir)s/dists/%(dist)s > %(cddir)s/dists/%(dist)s/Release" % {'dist':dist, 'cddir':cddir, 'temp_dir':temp_dir})
 313 if status != 0:
 314     print output
 315 assert status == 0
 316
 317 cmd = "gpg --default-key \"%(gpgkey)s\" --output %(cddir)s/dists/%(dist)s/Release.gpg -ba %(cddir)s/dists/%(dist)s/Release" % {'cddir':cddir, 'dist':dist, 'gpgkey':options.gpgkey}
 318 print cmd
 319 RunCommand(cmd)
 320
 321 # Clean up
 322 os.chdir( old_cwd )
 323 shutil.rmtree( temp_dir )
 324
 325
 326 # Find out the name of all the packages in the repository to add to the preseed file
 327 print "Generating a list of all the packages to install"
 328 package_names = []
 329 for deb in debs:
 330     status, output = commands.getstatusoutput( "dpkg --info \"%s\"" % deb )
 331     assert status == 0
 332     for line in output.split("\n"):
 333         line = line.rstrip("\n")
 334         splits = line.split(": ", 1)
 335         if len(splits) != 2:
 336             continue
 337         key, value = splits
 338         if key == " Package":
 339             package_names.append(value)
 340
 341 # now append the d-i directive to the preseed file to install our custom packages
 342 preseed_dir = os.path.join(options.cddir, 'preseed')
 343 camarabuntu_preseeds = [x for x in os.listdir(preseed_dir) if x.startswith('camarabuntu')]
 344 print repr(camarabuntu_preseeds)
 345 for preseed_file in camarabuntu_preseeds:
 346     preseed_file = os.path.join(preseed_dir, preseed_file)
 347     print 'editing ', preseed_file
 348     lines = open(preseed_file, 'r').readlines()
 349     newlines = []
 350     for line in lines:
 351         if line.startswith("d-i\tpkgsel/install-pattern\tstring ~t^edubuntu-standard$|~t^edubuntu-desktop$|~t^edubuntu-server"):
 352             print "Got to the line", line
 353             newlines.append( "# The following line has been automatically added by the camarabuntu scripts\n" )
 354             # we need to remove the trailing \n
 355             newlines.append( line[:-1]+"|"+"|".join( ["~n^%s$" % package_name for package_name in package_names] ) + "\n" )
 356         else:
 357             newlines.append(line)
 358
 359     newlines.append( "# The following line has been automatically added by the camarabuntu scripts\n" )
 360     newlines.append( "d-i\tpkgsel/include\tstring " + " ".join(package_names) + "\n" )
 361
 362     fd = open(preseed_file, "w")
 363     fd.write( "".join( newlines ) )
 364     fd.close()
 365
 366