recipes/iphone/iphone-sources

   1 #!/usr/bin/python
   2 #
   3 # Download Apple sources and prepare them for OpenEmbedded.
   4 #
   5 # You need the following on your system to run this script:
   6 #
   7 #  python
   8 #  python-beautifulsoup
   9 #  libssl-dev
  10 #  sudo (to mount HFS+ images)
  11 #
  12 # Version: 0.4
  13
  14 import cookielib
  15 import os
  16 import re
  17 import subprocess
  18 import shutil
  19 import sys
  20 import urllib
  21 import urllib2
  22 import urlparse
  23 from BeautifulSoup import BeautifulSoup
  24 import Cookie
  25 try:
  26     import xml.etree.cElementTree as ET
  27 except:
  28     import cElementTree as ET
  29
  30 DOWNLOAD_DIR = "downloads"
  31 MOUNT_DIR = "mnt"
  32 OUTPUT_DIR = "files"
  33 TEMP_DIR = "tmp"
  34 TOOLS_DIR = "tools"
  35
  36 class CustomCookieHandler(urllib2.BaseHandler):
  37     """
  38     Custom handler for cookies, as for some reason HTTPCookieProcessor
  39     has issues parsing the cookies received from Apple.
  40     """
  41     def __init__(self):
  42         self.cookiejar = Cookie.SimpleCookie()
  43
  44     def http_request(self, request):
  45         # add cookies
  46         attrs = []
  47         for key in self.cookiejar:
  48             attrs.append( key + "=" + self.cookiejar[key].value)
  49         if len(attrs):
  50             if not request.has_header("Cookie"):
  51                 request.add_unredirected_header(
  52                     "Cookie", "; ".join(attrs))
  53         return request
  54
  55     def http_response(self, request, response):
  56         # store cookies
  57         cookie = response.info().getheader('set-cookie')
  58         if cookie:
  59             self.cookiejar.load(cookie)
  60
  61         return response
  62
  63     https_request = http_request
  64     https_response = http_response
  65
  66 def plist_value(e):
  67     """
  68     Convert an XML element into its python representation.
  69     """
  70     if e.tag == "integer":
  71         return int(e.text)
  72     elif e.tag == "string":
  73         return e.text
  74     elif e.tag == "array":
  75         return [ plist_value(c) for c in e.getchildren() ]
  76     elif e.tag == "dict":
  77         key = None
  78         val = {}
  79         for c in e.getchildren():
  80             if c.tag == "key":
  81                 key = c.text
  82             else:
  83                 val[key] = plist_value(c)
  84         return val
  85     else:
  86         raise Exception("Could not parse dict entry %s" % e)
  87
  88 def plist_to_hash(plist_string):
  89     """
  90     Convert the contents of an Apple .plist file to a hash.
  91     """
  92     return plist_value(ET.fromstring(plist_string).find('dict'))
  93
  94 def extract_pkg(pkg, dest):
  95     print "Extracting package %s to %s" % (pkg, dest)
  96     if os.system("cd %s && xar -xf %s Payload && zcat Payload | cpio -id && rm Payload" % (dest, os.path.abspath(pkg))):
  97         raise Exception("Could not extract package")
  98
  99 def mount_dmg(dmg, mnt):
 100     """
 101     Convert a DMG image to an HFS+ image then mount it.
 102
 103     NOTE: requires sudo access
 104     """
 105     img = os.path.join(TEMP_DIR, os.path.basename(dmg).replace(".dmg", ".img"))
 106
 107     # Check image is not mounted
 108     if not os.path.exists(mnt):
 109         os.mkdir(mnt)
 110
 111     # Convert image
 112     if not os.path.exists(img):
 113         print "Converting image %s to %s " % (dmg, img)
 114         if os.system("%s -i %s -o %s" % (os.path.join(TOOLS_DIR, "dmg2img"), dmg, img)):
 115             raise Exception("Could not convert image")
 116
 117     # Mount image
 118     print "Mounting image %s on %s" % (img, mnt)
 119     if os.system("sudo mount -t hfsplus -o loop %s %s" % (img, mnt)):
 120         raise Exception("Could not mount image")
 121
 122     return img
 123
 124 def umount_dmg(dmg, mnt):
 125     """
 126     Unmount a DMG image.
 127
 128     NOTE: requires sudo access
 129     """
 130     img = os.path.join(TEMP_DIR, os.path.basename(dmg).replace(".dmg", ".img"))
 131     print "Unmounting %s" % (mnt)
 132     if os.system("sudo umount %s" % mnt):
 133         raise Exception("Could not unmount image")
 134     os.remove(img)
 135
 136 def pagetype(page):
 137      return page.info().getheader('content-type').split(';')[0]
 138
 139 def request(url, data=None):
 140     """
 141     Wrapper around urllibs2.urlopen to handle Apple's authentication form.
 142     """
 143     page = urllib2.urlopen(url, data)
 144
 145     if page.geturl() != url:
 146         # check for connect form
 147         soup = BeautifulSoup(page)
 148         form = soup.find("form", {"name": "appleConnectForm"})
 149         if form:
 150             login_url = urlparse.urljoin(page.geturl(), form['action'])
 151
 152             # log in
 153             print "* Logging into %s" % login_url
 154             page = urllib2.urlopen(urllib2.Request(login_url, data=urllib.urlencode({
 155                 'theAccountName': apple_id,
 156                 'theAccountPW': apple_password,
 157                 'theAuxValue': '',
 158                 '1.Continue.x': '1',
 159                 '1.Continue.y': '1'}),
 160                 headers={'Referer': page.geturl()}))
 161
 162     # follow refresh
 163     while pagetype(page) == "text/html":
 164         soup = BeautifulSoup(page)
 165         meta = soup.find('meta', {'http-equiv': 'REFRESH'})
 166         if not meta:
 167             break
 168
 169         refresh_url = meta['content'].split(';',1)[1].split("=",1)[1]
 170         print "* Following refresh to %s" % refresh_url
 171         page = urllib2.urlopen(urllib2.Request(refresh_url, data=data, headers={'Referer': page.geturl()}))
 172
 173     return page
 174
 175 def download(url, dest):
 176     """
 177     Download a file to a directory.
 178     """
 179     tempname = os.path.join(TEMP_DIR, os.path.basename(url)) + ".part"
 180     filename = os.path.join(dest, os.path.basename(url))
 181     if os.path.exists(filename):
 182         print "* Not downloading %s" % url
 183         return filename
 184
 185     print "* Downloading %s" % url
 186     page = request(url)
 187     size = page.info().getheader('content-length')
 188     if pagetype(page) == "text/html":
 189         raise Exception("We got an HTML page, download failed")
 190
 191     done = 0
 192     output = open(tempname, 'wb')
 193     sys.stdout.write("\r* Receiving..")
 194     sys.stdout.flush()
 195     while True:
 196         data = page.read(100000)
 197         if data:
 198             done += len(data)
 199             if size:
 200                 sys.stdout.write("\r* Received %i bytes (%f %%)" % (done, float(done * 100) / float(size)))
 201             else:
 202                 sys.stdout.write("\r* Received %i bytes (unknown)" % done)
 203             sys.stdout.flush()
 204             output.write(data)
 205         else:
 206             sys.stdout.write("\n")
 207             break
 208     output.close()
 209
 210     shutil.move(tempname, filename)
 211     return filename
 212
 213 def build_tools(url, binaries):
 214     """
 215     Build DMG handling tools.
 216     """
 217     print "[ Building tools ]"
 218     done = True
 219     for bin in binaries:
 220         tool = os.path.join(TOOLS_DIR, bin)
 221         if not os.path.exists(tool):
 222             print "* Tool %s is missing" % bin
 223             done = False
 224
 225     if done:
 226         print "* Tools already built"
 227         return
 228
 229     # download and extract tools
 230     workdir = os.path.join(TEMP_DIR, "tools")
 231     if os.path.exists(workdir):
 232         shutil.rmtree(workdir)
 233     os.mkdir(workdir)
 234     tarball = download(url, TEMP_DIR)
 235     if os.system("tar -C %s --strip-components=1 -xf %s" % (workdir, tarball)):
 236         raise Exception("Could not extract tarball")
 237
 238     # build tools
 239     if os.system("make -C %s" % workdir):
 240         raise Exception("Could not build tools")
 241     for bin in binaries:
 242         shutil.move(os.path.join(workdir, bin), os.path.join(TOOLS_DIR, bin))
 243
 244     # cleanup
 245     shutil.rmtree(workdir)
 246     os.remove(tarball)
 247
 248 def get_darwin_sources(urls):
 249     """
 250     Download the source .tar.gz of Darwin opensource components.
 251     """
 252     print "[ Darwin sources ]"
 253     for url in urls:
 254         download(url, OUTPUT_DIR)
 255
 256 def get_firmware_key(firmware_version, firmware_build):
 257     """
 258     Return the key for a given firmware by parsing the iPhone wiki.
 259     """
 260     page = urllib2.urlopen('http://www.theiphonewiki.com/wiki/index.php?title=VFDecrypt_Keys:_2.x')
 261     soup = BeautifulSoup(page)
 262
 263     for title in soup.findAll('span', {'class': 'mw-headline'}):
 264         v = title.contents[0].strip()
 265         if v.startswith(firmware_version) and v.count("(Build %s)" % firmware_build):
 266             p = title.parent.findNextSibling('p')
 267             return p.contents[0].strip().upper()
 268
 269     return None
 270
 271 def generate_iphone_rootfs(url, exclude):
 272     """
 273     Generate a .tar.bz2 for the iPhone rootfs.
 274     """
 275     print "[ iPhone rootfs ]"
 276     ipsw = download(url, DOWNLOAD_DIR)
 277
 278     # Get image contents
 279     plist = plist_to_hash(subprocess.Popen(["unzip", "-p", ipsw, "Restore.plist"], stdout=subprocess.PIPE).communicate()[0])
 280
 281     tarname = "iphone-rootfs-%s" % plist['ProductVersion']
 282     workdir = os.path.join(TEMP_DIR, tarname)
 283     tarball = os.path.join(OUTPUT_DIR, tarname + ".tar.bz2")
 284     tartemp = os.path.join(TEMP_DIR, tarname + ".tar.bz2")
 285     imagename = plist['SystemRestoreImages']['User']
 286     image = os.path.join(TEMP_DIR, imagename)
 287
 288     if os.path.exists(tarball):
 289         print "Skipping %s" % tarball
 290         return
 291
 292     for i in ['ProductType', 'ProductVersion', 'ProductBuildVersion']:
 293         print "\t%s: %s" % (i, plist[i])
 294
 295     # Get firmware key
 296     print "Fetching decryption key for %s" % image
 297     key = get_firmware_key(plist['ProductVersion'], plist['ProductBuildVersion'])
 298     if not key:
 299         raise Exception("Could not get decryption key for firmware")
 300     print "Got decryption key %s" % key
 301
 302     # Get dmg
 303     if not os.path.exists(image):
 304         # Extract
 305         print "Extracting %s" % image
 306         os.system("unzip %s %s -d %s" % (ipsw, imagename, TEMP_DIR))
 307
 308         # Decrypt
 309         temp = image + ".decrypted"
 310         if os.system("%s -k %s -i %s -o %s" % (os.path.join(TOOLS_DIR, "vfdecrypt"), key, image, temp)):
 311             raise Exception("Could not decrypt image")
 312         os.remove(image)
 313         os.rename(temp, image)
 314
 315     # Generate tarball
 316     os.mkdir(workdir)
 317     mount_dmg(image, workdir)
 318
 319     print "Generating %s" % tarball
 320     excludes = " ".join([ "--exclude %s/%s" % (tarname, x) for x in exclude ])
 321     if os.system("tar %s --hard-dereference -C %s -cjf %s %s" % (excludes, TEMP_DIR, tartemp, tarname)):
 322         raise Exception("Could not create tarball")
 323     shutil.move(tartemp, tarball)
 324
 325     umount_dmg(image, workdir)
 326     os.rmdir(workdir)
 327     os.remove(image)
 328     return tarball
 329
 330 def generate_iphone_sdk(url, iphone_version, macosx_version):
 331     """
 332     Generate a .tar.bz2 for the iPhone SDKs.
 333     """
 334     print "[ iPhone SDKs ]"
 335     #request('http://developer.apple.com/iphone/login.action')
 336     image = download(url, DOWNLOAD_DIR)
 337
 338     tarname = "iphone-sdks-%s" % iphone_version
 339     workdir = os.path.join(TEMP_DIR, tarname)
 340     tarball = os.path.join(OUTPUT_DIR, tarname + ".tar.bz2")
 341     tartemp = os.path.join(TEMP_DIR, tarname + ".tar.bz2")
 342
 343     if os.path.exists(tarball):
 344         print "Skipping %s" % tarball
 345         return
 346
 347     # Cleanup
 348     if os.path.exists(workdir):
 349         shutil.rmtree(workdir)
 350
 351     # Mount image
 352     mount_dmg(image, MOUNT_DIR)
 353     os.mkdir(workdir)
 354
 355     # Extract iPhone stuff
 356     sdk = "iPhoneOS%s.sdk" % iphone_version
 357     tmpdir = os.path.join(TEMP_DIR, sdk)
 358     os.mkdir(tmpdir)
 359     extract_pkg("%s/Packages/iPhoneSDKHeadersAndLibs.pkg" % MOUNT_DIR, tmpdir)
 360     shutil.move("%s/Platforms/iPhoneOS.platform/Developer/SDKs/%s" % (tmpdir, sdk), os.path.join(workdir, sdk))
 361     shutil.rmtree(tmpdir)
 362
 363     # Extract MacOS stuff
 364     sdk = "MacOSX%s.sdk" % macosx_version
 365     tmpdir = os.path.join(TEMP_DIR, sdk)
 366     os.mkdir(tmpdir)
 367     extract_pkg("%s/Packages/MacOSX%s.pkg" % (MOUNT_DIR, macosx_version), tmpdir)
 368     shutil.move("%s/SDKs/%s" % (tmpdir, sdk), os.path.join(workdir, sdk))
 369     shutil.rmtree(tmpdir)
 370
 371     # Unmount image
 372     umount_dmg(image, MOUNT_DIR)
 373
 374     print "Generating %s" % tarball
 375     if os.system("tar -C %s -cjf %s %s" % (TEMP_DIR, tartemp, tarname)):
 376         raise Exception("Could not create tarball")
 377     shutil.move(tartemp, tarball)
 378     shutil.rmtree(workdir)
 379
 380     return tarball
 381
 382 if __name__ == "__main__":
 383     # Check arguments
 384     if len(sys.argv) < 4:
 385         print """Usage: iphone-sources <manifest> <apple_id> <apple_password>"""
 386         sys.exit(1)
 387     cmd = sys.argv[1]
 388
 389     # Initialise directories
 390     for d in [DOWNLOAD_DIR, OUTPUT_DIR, MOUNT_DIR, TEMP_DIR, TOOLS_DIR]:
 391         if not os.path.exists(d):
 392             print "Creating %s" % d
 393             os.mkdir(d)
 394
 395     # Read manifest
 396     manifest = eval(file(sys.argv[1]).read())
 397     apple_id = sys.argv[2]
 398     apple_password = sys.argv[3]
 399
 400     # Register cookies
 401     opener = urllib2.build_opener(CustomCookieHandler())
 402     urllib2.install_opener(opener)
 403
 404     # Perform all build steps
 405     build_tools(manifest['tools']['url'], manifest['tools']['binaries'])
 406     get_darwin_sources(manifest['sources'])
 407     generate_iphone_rootfs(manifest['firmware']['url'], manifest['firmware']['exclude'])
 408     generate_iphone_sdk(manifest['sdk']['url'], manifest['firmware']['version'], manifest['macosx']['version'])
 409
 410     # Cleanup
 411     for d in [MOUNT_DIR, TEMP_DIR]:
 412         os.rmdir(d)
 413