have different dists
[camarabuntu.git] / bin / add-debs-to-cd-image.py
blobe7f9894e640e4df55b20b5176d68c973b6f41853
1 #! /usr/bin/python
3 from optparse import OptionParser
4 import os, shutil, tempfile, commands, glob, sys, re, subprocess, shutil, time
6 usage = "%prog [options] <list of .debs>"
8 parser = OptionParser(usage=usage)
10 parser.add_option("-d", "--cd-dir",
11 dest="cddir", default=None,
12 help="The directory of the install cd details")
14 parser.add_option( "-k", "--gpg-key",
15 dest="gpgkey", default=None, type="string",
16 help="The GPG key used to sign the packages" )
18 parser.add_option( "--ubuntu-keyring",
19 dest="keyring", default=None, type="string",
20 help="The directory of the ubuntu keyring source. if not provided it will be downloaded." )
22 parser.add_option( "--indices",
23 dest="indices", default=None, type="string",
24 help="The directory of the ubuntu indices" )
26 parser.add_option( "--passphrase",
27 dest="passphrase", default=None, type="string",
28 help="The passphrase for the gpg key" )
30 parser.add_option( "--dist",
31 dest="dist", default=None, type="string",
32 help="Instead of autodetecting the dist, use this" )
34 (options, orig_debs) = parser.parse_args()
36 if not orig_debs or not options.cddir or not options.gpgkey or not options.indices:
37 parser.print_help()
38 sys.exit(1)
40 assert os.path.isdir( options.indices )
42 debs = [os.path.abspath( deb ) for deb in orig_debs ]
44 cddir = os.path.abspath( options.cddir )
45 indices = os.path.abspath( options.indices )
47 if options.dist is None:
48 # see what the name of the distro is, (eg dapper, edgy.. )
49 dists_dir = [ dir for dir in os.listdir( os.path.join ( cddir, 'dists' ) ) if dir not in ['stable', 'unstable' ] ]
51 assert len(dists_dir) == 1
52 dist = dists_dir[0]
53 else:
54 dist = options.dist
56 dist_name_to_version = { 'warty':'4.10', 'hoary':'5.04', 'breezy':'5.10',
57 'dapper':'6.06', 'edgy':'6.10', 'feisty':'7.04',
58 'gutsy':'7.10', 'hardy':'8.04', 'intrepid':'8.10' }
60 assert dist in dist_name_to_version.keys()
62 def RunCommand(cmd, msg=None):
63 print "running : %s" % cmd
64 p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
65 returncode = p.wait()
66 print "output : ", p.stdout.read()
67 print "stderr: ", p.stderr.read()
68 if returncode != 0:
69 print "FAILED!"
70 if msg is not None: print msg
71 sys.exit(returncode)
73 def makedir_if_not_exist( *path_parts ):
74 path = os.path.join( *path_parts )
75 if not os.path.exists( path ):
76 os.makedirs( path )
78 makedir_if_not_exist( cddir, 'dists', dist, 'extras', 'binary-i386' )
79 makedir_if_not_exist( cddir, 'pool', 'extras' )
80 makedir_if_not_exist( cddir, 'isolinux' )
81 makedir_if_not_exist( cddir, 'preseed' )
83 # Copy all the debs to the extras file
84 for deb in debs:
85 print "Copying %s" % deb
86 shutil.copy( deb, os.path.join( cddir, "pool", "extras" ) )
88 releases_file = open( os.path.join( cddir, 'dists', dist, 'extras', 'binary-i386', 'Release' ), 'w' )
89 releases_file.write( "Archive: " + dist + "\n" )
90 releases_file.write( "Version: " + dist_name_to_version[dist] + "\n" )
91 releases_file.write( "Component: extras\nOrigin: Ubuntu\nLabel: Ubuntu\nArchitecture: i386\n")
92 releases_file.close()
93 print "Wrote the Releases file"
96 old_cwd = os.getcwd()
97 temp_dir = tempfile.mkdtemp( prefix="/tmp/camarabuntu-tmp-", dir=old_cwd)
98 os.chdir( temp_dir )
100 #shutil.copytree( os.path.join( old_cwd, cddir, 'pool', 'main', 'u', 'ubuntu-keyring' ), temp_dir )
101 #assert false;
104 # TODO different versions of ubuntu-keyring
105 if options.keyring is None:
106 # download a new one
107 print "Downloading the ubuntu-keyring..."
108 cmd = "apt-get source ubuntu-keyring"
109 msg = """An error occured when downloading the source of the ubuntu-keyring package
110 This is needed to sign our new packages"""
111 RunCommand(cmd, msg=msg)
112 else:
113 if options.keyring[-1] == '/':
114 # This should be a directory, if there's a / at the end, it'll break os.path.split
115 options.keyring = options.keyring[0:-1]
116 shutil.copytree( os.path.join( old_cwd, options.keyring ), os.path.join( temp_dir, os.path.split( options.keyring )[1] ) )
118 # find the file
119 gpg_keys_filename = glob.glob ( os.path.join( temp_dir, 'ubuntu-keyring*/keyrings/ubuntu-archive-keyring.gpg' ) )[0]
120 ubuntu_keyring_dir = [name for name in glob.glob( os.path.join( temp_dir, 'ubuntu-keyring*' ) ) if os.path.isdir(name)][0]
122 assert os.path.isfile( gpg_keys_filename )
124 print "Adding GPG key %s to the ubuntu-keyring" % options.gpgkey
126 cmd = "gpg --import < %s" % gpg_keys_filename
127 # if this fails someone was messing with the code before
128 RunCommand(cmd)
130 cmd = "gpg --export FBB75451 437D05B5 %s > %s" % (options.gpgkey, gpg_keys_filename)
131 # Invalid keys are not detected here, unfortunatly
132 RunCommand(cmd)
134 print "\n\nRebuilding the ubuntu-keyring."
135 os.chdir( ubuntu_keyring_dir )
136 # Invalid keys are not detected here, unfortunatly
137 cmd = "dpkg-buildpackage -rfakeroot -k%s" % options.gpgkey
138 msg = """Unable to rebuild the ubuntu-keyring
139 Possible causes:
140 (*) The GPG key you gave (%s) is invalid, check available keys with "gpg --list-keys" """ % options.gpgkey
141 RunCommand(cmd, msg=msg)
142 print "Finished Rebuilding the ubuntu-keyring.\n\n"
144 # Copy these files into the main component
145 shutil.copy( glob.glob( os.path.join( temp_dir, "ubuntu-keyring*_all.deb" ) )[0], os.path.join( old_cwd, cddir, 'pool/main/u/ubuntu-keyring/' ) )
148 # Clean up, remove all this crap
149 os.chdir( old_cwd )
150 shutil.rmtree( temp_dir )
153 ## Now create the indices.
154 temp_dir = tempfile.mkdtemp( prefix="/tmp/camarabuntu-tmp-", dir=old_cwd )
155 os.chdir( temp_dir )
157 ftparchive_deb = open( os.path.join( temp_dir, 'apt-ftparchive-deb.conf' ), 'w' )
158 ftparchive_deb.write( """Dir {
159 ArchiveDir "%(cddir)s";
162 TreeDefault {
163 Directory "pool/";
166 BinDirectory "pool/main" {
167 Packages "dists/%(dist)s/main/binary-i386/Packages";
168 BinOverride "%(indices)s/override.%(dist)s.main";
169 ExtraOverride "%(indices)s/override.%(dist)s.extra.main";
172 BinDirectory "pool/restricted" {
173 Packages "dists/%(dist)s/restricted/binary-i386/Packages";
174 BinOverride "%(indices)s/override.%(dist)s.restricted";
177 Default {
178 Packages {
179 Extensions ".deb";
180 Compress ". gzip";
184 Contents {
185 Compress "gzip";
187 """ % {'cddir':cddir, 'dist':dist, 'indices':indices} )
189 ftparchive_deb.close()
191 # generate the extras override. This is converted from the perl version on
192 # https://help.ubuntu.com/community/InstallCDCustomization
194 main_packages = open( os.path.join( old_cwd, cddir, 'dists', dist, 'main/binary-i386/Packages' ) )
195 extra_overrides = open( os.path.join( temp_dir, "override.%s.extra.main" % dist ), 'w' )
196 task = None
197 for line in main_packages:
198 line = line.rstrip("\n")
199 line = line.rstrip("\r\n")
200 line = line.rstrip("\r")
201 if re.search("^\s", line):
202 continue # to the next line
203 if line == "" and task is not None:
204 extra_overrides.write("%s Task %s\n" % (package, task))
205 package = None
206 task = None
207 if line == "":
208 continue
209 key, value = line.split(": ", 1)
210 if key == "Package":
211 package = value
212 if key == "Task":
213 task = value
216 extra_overrides.close()
217 main_packages.close()
219 ftparchive_udeb = open( os.path.join( temp_dir, "apt-ftparchive-udeb.conf" ), 'w')
220 ftparchive_udeb.write("""Dir {
221 ArchiveDir "%(cddir)s";
224 TreeDefault {
225 Directory "pool/";
228 BinDirectory "pool/main" {
229 Packages "dists/%(dist)s/main/debian-installer/binary-i386/Packages";
230 BinOverride "%(indices)s/override.%(dist)s.main.debian-installer";
233 BinDirectory "pool/restricted" {
234 Packages "dists/%(dist)s/restricted/debian-installer/binary-i386/Packages";
235 BinOverride "%(indices)s/override.%(dist)s.restricted.debian-installer";
238 Default {
239 Packages {
240 Extensions ".udeb";
241 Compress ". gzip";
245 Contents {
246 Compress "gzip";
248 """ % {'cddir':cddir, 'dist':dist, 'indices':indices} )
249 ftparchive_udeb.close()
251 ftparchive_extras = open( os.path.join( temp_dir, 'apt-ftparchive-extras.conf' ), 'w')
252 ftparchive_extras.write("""Dir {
253 ArchiveDir "%(cddir)s";
256 TreeDefault {
257 Directory "pool/";
260 BinDirectory "pool/extras" {
261 Packages "dists/%(dist)s/extras/binary-i386/Packages";
264 Default {
265 Packages {
266 Extensions ".deb";
267 Compress ". gzip";
271 Contents {
272 Compress "gzip";
274 """ % {'cddir':cddir, 'dist':dist, 'indices':indices} )
275 ftparchive_extras.close()
277 release_file = open( os.path.join( temp_dir, "release.conf" ), 'w' )
278 release_file.write("""
279 APT::FTPArchive::Release::Origin "Ubuntu";
280 APT::FTPArchive::Release::Label "Ubuntu";
281 APT::FTPArchive::Release::Suite "%(dist)s";
282 APT::FTPArchive::Release::Version "%(dist_number)s";
283 APT::FTPArchive::Release::Codename "%(dist)s";
284 APT::FTPArchive::Release::Architectures "i386";
285 APT::FTPArchive::Release::Components "main restricted extras";
286 APT::FTPArchive::Release::Description "Ubuntu %(dist_number)s";
287 """ % {'cddir':cddir, 'dist':dist, 'indices':indices, 'dist_number':dist_name_to_version[dist]} )
288 release_file.close()
290 # now build the repository
292 print "Generating the APT repository..."
294 status, output = commands.getstatusoutput("apt-ftparchive -c %(temp_dir)s/release.conf generate %(temp_dir)s/apt-ftparchive-deb.conf" % {'temp_dir':temp_dir})
295 if status != 0:
296 print output
297 assert status == 0
299 status, output = commands.getstatusoutput("apt-ftparchive -c %(temp_dir)s/release.conf generate %(temp_dir)s/apt-ftparchive-udeb.conf" % {'temp_dir':temp_dir})
300 if status != 0:
301 print output
302 assert status == 0
304 status, output = commands.getstatusoutput("apt-ftparchive -c %(temp_dir)s/release.conf generate %(temp_dir)s/apt-ftparchive-extras.conf"% {'temp_dir':temp_dir})
305 if status != 0:
306 print output
307 assert status == 0
309 if os.path.isfile( os.path.join( cddir, 'dists', dist, 'Release.gpg') ):
310 # Otherwise apt-ftparchive will ask to overwrite it, we want this whole process automated
311 os.remove( os.path.join( cddir, 'dists', dist, 'Release.gpg' ) )
312 status, output = commands.getstatusoutput("apt-ftparchive -c %(temp_dir)s/release.conf release %(cddir)s/dists/%(dist)s > %(cddir)s/dists/%(dist)s/Release" % {'dist':dist, 'cddir':cddir, 'temp_dir':temp_dir})
313 if status != 0:
314 print output
315 assert status == 0
317 cmd = "gpg --default-key \"%(gpgkey)s\" --output %(cddir)s/dists/%(dist)s/Release.gpg -ba %(cddir)s/dists/%(dist)s/Release" % {'cddir':cddir, 'dist':dist, 'gpgkey':options.gpgkey}
318 print cmd
319 RunCommand(cmd)
321 # Clean up
322 os.chdir( old_cwd )
323 shutil.rmtree( temp_dir )
326 # Find out the name of all the packages in the repository to add to the preseed file
327 print "Generating a list of all the packages to install"
328 package_names = []
329 for deb in debs:
330 status, output = commands.getstatusoutput( "dpkg --info \"%s\"" % deb )
331 assert status == 0
332 for line in output.split("\n"):
333 line = line.rstrip("\n")
334 splits = line.split(": ", 1)
335 if len(splits) != 2:
336 continue
337 key, value = splits
338 if key == " Package":
339 package_names.append(value)
341 # now append the d-i directive to the preseed file to install our custom packages
342 preseed_dir = os.path.join(options.cddir, 'preseed')
343 camarabuntu_preseeds = [x for x in os.listdir(preseed_dir) if x.startswith('camarabuntu')]
344 print repr(camarabuntu_preseeds)
345 for preseed_file in camarabuntu_preseeds:
346 preseed_file = os.path.join(preseed_dir, preseed_file)
347 print 'editing ', preseed_file
348 lines = open(preseed_file, 'r').readlines()
349 newlines = []
350 for line in lines:
351 if line.startswith("d-i\tpkgsel/install-pattern\tstring ~t^edubuntu-standard$|~t^edubuntu-desktop$|~t^edubuntu-server"):
352 print "Got to the line", line
353 newlines.append( "# The following line has been automatically added by the camarabuntu scripts\n" )
354 # we need to remove the trailing \n
355 newlines.append( line[:-1]+"|"+"|".join( ["~n^%s$" % package_name for package_name in package_names] ) + "\n" )
356 else:
357 newlines.append(line)
359 newlines.append( "# The following line has been automatically added by the camarabuntu scripts\n" )
360 newlines.append( "d-i\tpkgsel/include\tstring " + " ".join(package_names) + "\n" )
362 fd = open(preseed_file, "w")
363 fd.write( "".join( newlines ) )
364 fd.close()