taskcluster/scripts/misc/repack_rust.py

   1 #!/usr/bin/env python3
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 """
   7 This script downloads and repacks official rust language builds
   8 with the necessary tool and target support for the Firefox
   9 build environment.
  10 """
  11
  12 import argparse
  13 import errno
  14 import hashlib
  15 import os
  16 import shutil
  17 import subprocess
  18 import tarfile
  19 import tempfile
  20 import textwrap
  21 from contextlib import contextmanager
  22
  23 import requests
  24 import toml
  25 import zstandard
  26
  27
  28 def log(msg):
  29     print("repack: %s" % msg, flush=True)
  30
  31
  32 def fetch_file(url):
  33     """Download a file from the given url if it's not already present.
  34
  35     Returns the SHA-2 256-bit hash of the received file."""
  36     filename = os.path.basename(url)
  37     sha = hashlib.sha256()
  38     size = 4096
  39     if os.path.exists(filename):
  40         with open(filename, "rb") as fd:
  41             while True:
  42                 block = fd.read(size)
  43                 if not block:
  44                     return sha.hexdigest()
  45                 sha.update(block)
  46         log("Could not calculate checksum!")
  47         return None
  48     r = requests.get(url, stream=True)
  49     r.raise_for_status()
  50     with open(filename, "wb") as fd:
  51         for chunk in r.iter_content(size):
  52             fd.write(chunk)
  53             sha.update(chunk)
  54         return sha.hexdigest()
  55
  56
  57 def check_call_with_input(cmd, input_data):
  58     """Invoke a command, passing the input String over stdin.
  59
  60     This is like subprocess.check_call, but allows piping
  61     input to interactive commands."""
  62     p = subprocess.Popen(cmd, stdin=subprocess.PIPE)
  63     p.communicate(input_data)
  64     if p.wait():
  65         raise subprocess.CalledProcessError(p.returncode, cmd)
  66
  67
  68 def setup_gpg():
  69     """Add the signing key to the current gpg config.
  70
  71     Import a hard-coded copy of the release signing public key
  72     and mark it trusted in the gpg database so subsequent
  73     signature checks can succeed or fail cleanly."""
  74     keyid = "0x85AB96E6FA1BE5FE"
  75     log("Importing signing key %s..." % keyid)
  76     key = b"""
  77 -----BEGIN PGP PUBLIC KEY BLOCK-----
  78
  79 mQINBFJEwMkBEADlPACa2K7reD4x5zd8afKx75QYKmxqZwywRbgeICeD4bKiQoJZ
  80 dUjmn1LgrGaXuBMKXJQhyA34e/1YZel/8et+HPE5XpljBfNYXWbVocE1UMUTnFU9
  81 CKXa4AhJ33f7we2/QmNRMUifw5adPwGMg4D8cDKXk02NdnqQlmFByv0vSaArR5kn
  82 gZKnLY6o0zZ9Buyy761Im/ShXqv4ATUgYiFc48z33G4j+BDmn0ryGr1aFdP58tHp
  83 gjWtLZs0iWeFNRDYDje6ODyu/MjOyuAWb2pYDH47Xu7XedMZzenH2TLM9yt/hyOV
  84 xReDPhvoGkaO8xqHioJMoPQi1gBjuBeewmFyTSPS4deASukhCFOcTsw/enzJagiS
  85 ZAq6Imehduke+peAL1z4PuRmzDPO2LPhVS7CDXtuKAYqUV2YakTq8MZUempVhw5n
  86 LqVaJ5/XiyOcv405PnkT25eIVVVghxAgyz6bOU/UMjGQYlkUxI7YZ9tdreLlFyPR
  87 OUL30E8q/aCd4PGJV24yJ1uit+yS8xjyUiMKm4J7oMP2XdBN98TUfLGw7SKeAxyU
  88 92BHlxg7yyPfI4TglsCzoSgEIV6xoGOVRRCYlGzSjUfz0bCMCclhTQRBkegKcjB3
  89 sMTyG3SPZbjTlCqrFHy13e6hGl37Nhs8/MvXUysq2cluEISn5bivTKEeeQARAQAB
  90 tERSdXN0IExhbmd1YWdlIChUYWcgYW5kIFJlbGVhc2UgU2lnbmluZyBLZXkpIDxy
  91 dXN0LWtleUBydXN0LWxhbmcub3JnPokCOAQTAQIAIgUCUkTAyQIbAwYLCQgHAwIG
  92 FQgCCQoLBBYCAwECHgECF4AACgkQhauW5vob5f5fYQ//b1DWK1NSGx5nZ3zYZeHJ
  93 9mwGCftIaA2IRghAGrNf4Y8DaPqR+w1OdIegWn8kCoGfPfGAVW5XXJg+Oxk6QIaD
  94 2hJojBUrq1DALeCZVewzTVw6BN4DGuUexsc53a8DcY2Yk5WE3ll6UKq/YPiWiPNX
  95 9r8FE2MJwMABB6mWZLqJeg4RCrriBiCG26NZxGE7RTtPHyppoVxWKAFDiWyNdJ+3
  96 UnjldWrT9xFqjqfXWw9Bhz8/EoaGeSSbMIAQDkQQpp1SWpljpgqvctZlc5fHhsG6
  97 lmzW5RM4NG8OKvq3UrBihvgzwrIfoEDKpXbk3DXqaSs1o81NH5ftVWWbJp/ywM9Q
  98 uMC6n0YWiMZMQ1cFBy7tukpMkd+VPbPkiSwBhPkfZIzUAWd74nanN5SKBtcnymgJ
  99 +OJcxfZLiUkXRj0aUT1GLA9/7wnikhJI+RvwRfHBgrssXBKNPOfXGWajtIAmZc2t
 100 kR1E8zjBVLId7r5M8g52HKk+J+y5fVgJY91nxG0zf782JjtYuz9+knQd55JLFJCO
 101 hhbv3uRvhvkqgauHagR5X9vCMtcvqDseK7LXrRaOdOUDrK/Zg/abi5d+NIyZfEt/
 102 ObFsv3idAIe/zpU6xa1nYNe3+Ixlb6mlZm3WCWGxWe+GvNW/kq36jZ/v/8pYMyVO
 103 p/kJqnf9y4dbufuYBg+RLqC5Ag0EUkTAyQEQANxy2tTSeRspfrpBk9+ju+KZ3zc4
 104 umaIsEa5DxJ2zIKHywVAR67Um0K1YRG07/F5+tD9TIRkdx2pcmpjmSQzqdk3zqa9
 105 2Zzeijjz2RNyBY8qYmyE08IncjTsFFB8OnvdXcsAgjCFmI1BKnePxrABL/2k8X18
 106 aysPb0beWqQVsi5FsSpAHu6k1kaLKc+130x6Hf/YJAjeo+S7HeU5NeOz3zD+h5bA
 107 Q25qMiVHX3FwH7rFKZtFFog9Ogjzi0TkDKKxoeFKyADfIdteJWFjOlCI9KoIhfXq
 108 Et9JMnxApGqsJElJtfQjIdhMN4Lnep2WkudHAfwJ/412fe7wiW0rcBMvr/BlBGRY
 109 vM4sTgN058EwIuY9Qmc8RK4gbBf6GsfGNJjWozJ5XmXElmkQCAvbQFoAfi5TGfVb
 110 77QQrhrQlSpfIYrvfpvjYoqj618SbU6uBhzh758gLllmMB8LOhxWtq9eyn1rMWyR
 111 KL1fEkfvvMc78zP+Px6yDMa6UIez8jZXQ87Zou9EriLbzF4QfIYAqR9LUSMnLk6K
 112 o61tSFmFEDobC3tc1jkSg4zZe/wxskn96KOlmnxgMGO0vJ7ASrynoxEnQE8k3WwA
 113 +/YJDwboIR7zDwTy3Jw3mn1FgnH+c7Rb9h9geOzxKYINBFz5Hd0MKx7kZ1U6WobW
 114 KiYYxcCmoEeguSPHABEBAAGJAh8EGAECAAkFAlJEwMkCGwwACgkQhauW5vob5f7f
 115 FA//Ra+itJF4NsEyyhx4xYDOPq4uj0VWVjLdabDvFjQtbBLwIyh2bm8uO3AY4r/r
 116 rM5WWQ8oIXQ2vvXpAQO9g8iNlFez6OLzbfdSG80AG74pQqVVVyCQxD7FanB/KGge
 117 tAoOstFxaCAg4nxFlarMctFqOOXCFkylWl504JVIOvgbbbyj6I7qCUmbmqazBSMU
 118 K8c/Nz+FNu2Uf/lYWOeGogRSBgS0CVBcbmPUpnDHLxZWNXDWQOCxbhA1Uf58hcyu
 119 036kkiWHh2OGgJqlo2WIraPXx1cGw1Ey+U6exbtrZfE5kM9pZzRG7ZY83CXpYWMp
 120 kyVXNWmf9JcIWWBrXvJmMi0FDvtgg3Pt1tnoxqdilk6yhieFc8LqBn6CZgFUBk0t
 121 NSaWk3PsN0N6Ut8VXY6sai7MJ0Gih1gE1xadWj2zfZ9sLGyt2jZ6wK++U881YeXA
 122 ryaGKJ8sIs182hwQb4qN7eiUHzLtIh8oVBHo8Q4BJSat88E5/gOD6IQIpxc42iRL
 123 T+oNZw1hdwNyPOT1GMkkn86l3o7klwmQUWCPm6vl1aHp3omo+GHC63PpNFO5RncJ
 124 Ilo3aBKKmoE5lDSMGE8KFso5awTo9z9QnVPkRsk6qeBYit9xE3x3S+iwjcSg0nie
 125 aAkc0N00nc9V9jfPvt4z/5A5vjHh+NhFwH5h2vBJVPdsz6m5Ag0EVI9keAEQAL3R
 126 oVsHncJTmjHfBOV4JJsvCum4DuJDZ/rDdxauGcjMUWZaG338ZehnDqG1Yn/ys7zE
 127 aKYUmqyT+XP+M2IAQRTyxwlU1RsDlemQfWrESfZQCCmbnFScL0E7cBzy4xvtInQe
 128 UaFgJZ1BmxbzQrx+eBBdOTDv7RLnNVygRmMzmkDhxO1IGEu1+3ETIg/DxFE7VQY0
 129 It/Ywz+nHu1o4Hemc/GdKxu9hcYvcRVc/Xhueq/zcIM96l0m+CFbs0HMKCj8dgMe
 130 Ng6pbbDjNM+cV+5BgpRdIpE2l9W7ImpbLihqcZt47J6oWt/RDRVoKOzRxjhULVyV
 131 2VP9ESr48HnbvxcpvUAEDCQUhsGpur4EKHFJ9AmQ4zf91gWLrDc6QmlACn9o9ARU
 132 fOV5aFsZI9ni1MJEInJTP37stz/uDECRie4LTL4O6P4Dkto8ROM2wzZq5CiRNfnT
 133 PP7ARfxlCkpg+gpLYRlxGUvRn6EeYwDtiMQJUQPfpGHSvThUlgDEsDrpp4SQSmdA
 134 CB+rvaRqCawWKoXs0In/9wylGorRUupeqGC0I0/rh+f5mayFvORzwy/4KK4QIEV9
 135 aYTXTvSRl35MevfXU1Cumlaqle6SDkLr3ZnFQgJBqap0Y+Nmmz2HfO/pohsbtHPX
 136 92SN3dKqaoSBvzNGY5WT3CsqxDtik37kR3f9/DHpABEBAAGJBD4EGAECAAkFAlSP
 137 ZHgCGwICKQkQhauW5vob5f7BXSAEGQECAAYFAlSPZHgACgkQXLSpNHs7CdwemA/+
 138 KFoGuFqU0uKT9qblN4ugRyil5itmTRVffl4tm5OoWkW8uDnu7Ue3vzdzy+9NV8X2
 139 wRG835qjXijWP++AGuxgW6LB9nV5OWiKMCHOWnUjJQ6pNQMAgSN69QzkFXVF/q5f
 140 bkma9TgSbwjrVMyPzLSRwq7HsT3V02Qfr4cyq39QeILGy/NHW5z6LZnBy3BaVSd0
 141 lGjCEc3yfH5OaB79na4W86WCV5n4IT7cojFM+LdL6P46RgmEtWSG3/CDjnJl6BLR
 142 WqatRNBWLIMKMpn+YvOOL9TwuP1xbqWr1vZ66wksm53NIDcWhptpp0KEuzbU0/Dt
 143 OltBhcX8tOmO36LrSadX9rwckSETCVYklmpAHNxPml011YNDThtBidvsicw1vZwR
 144 HsXn+txlL6RAIRN+J/Rw3uOiJAqN9Qgedpx2q+E15t8MiTg/FXtB9SysnskFT/BH
 145 z0USNKJUY0btZBw3eXWzUnZf59D8VW1M/9JwznCHAx0c9wy/gRDiwt9w4RoXryJD
 146 VAwZg8rwByjldoiThUJhkCYvJ0R3xH3kPnPlGXDW49E9R8C2umRC3cYOL4U9dOQ1
 147 5hSlYydF5urFGCLIvodtE9q80uhpyt8L/5jj9tbwZWv6JLnfBquZSnCGqFZRfXlb
 148 Jphk9+CBQWwiZSRLZRzqQ4ffl4xyLuolx01PMaatkQbRaw/+JpgRNlurKQ0PsTrO
 149 8tztO/tpBBj/huc2DGkSwEWvkfWElS5RLDKdoMVs/j5CLYUJzZVikUJRm7m7b+OA
 150 P3W1nbDhuID+XV1CSBmGifQwpoPTys21stTIGLgznJrIfE5moFviOLqD/LrcYlsq
 151 CQg0yleu7SjOs//8dM3mC2FyLaE/dCZ8l2DCLhHw0+ynyRAvSK6aGCmZz6jMjmYF
 152 MXgiy7zESksMnVFMulIJJhR3eB0wx2GitibjY/ZhQ7tD3i0yy9ILR07dFz4pgkVM
 153 afxpVR7fmrMZ0t+yENd+9qzyAZs0ksxORoc2ze90SCx2jwEX/3K+m4I0hP2H/w5W
 154 gqdvuRLiqf+4BGW4zqWkLLlNIe/okt0r82SwHtDN0Ui1asmZTGj6sm8SXtwx+5cE
 155 38MttWqjDiibQOSthRVcETByRYM8KcjYSUCi4PoBc3NpDONkFbZm6XofR/f5mTcl
 156 2jDw6fIeVc4Hd1jBGajNzEqtneqqbdAkPQaLsuD2TMkQfTDJfE/IljwjrhDa9Mi+
 157 odtnMWq8vlwOZZ24/8/BNK5qXuCYL67O7AJB4ZQ6BT+g4z96iRLbupzu/XJyXkQF
 158 rOY/Ghegvn7fDrnt2KC9MpgeFBXzUp+k5rzUdF8jbCx5apVjA1sWXB9Kh3L+DUwF
 159 Mve696B5tlHyc1KxjHR6w9GRsh4=
 160 =5FXw
 161 -----END PGP PUBLIC KEY BLOCK-----
 162 """
 163     check_call_with_input(["gpg", "--import"], key)
 164     check_call_with_input(
 165         ["gpg", "--command-fd", "0", "--edit-key", keyid], b"trust\n5\ny\n"
 166     )
 167
 168
 169 def verify_sha(filename, sha):
 170     """Verify that the checksum file matches the given sha digest."""
 171     sha_filename = filename + ".sha256"
 172     with open(sha_filename) as f:
 173         # Older sha256 files would contain `sha filename`, but more recent
 174         # ones only contain `sha`.
 175         checksum = f.readline().split()[0]
 176         if checksum != sha:
 177             raise ValueError("Checksum mismatch in %s" % filename)
 178         return True
 179     log("No checksum file for %s!" % filename)
 180     return False
 181
 182
 183 def fetch(url, validate=True):
 184     """Download and verify a package url."""
 185     base = os.path.basename(url)
 186     log("Fetching %s..." % base)
 187     if validate:
 188         fetch_file(url + ".asc")
 189         fetch_file(url + ".sha256")
 190     sha = fetch_file(url)
 191     if validate:
 192         log("Verifying %s..." % base)
 193         verify_sha(base, sha)
 194         subprocess.check_call(
 195             ["gpg", "--keyid-format", "0xlong", "--verify", base + ".asc", base]
 196         )
 197     return sha
 198
 199
 200 def install(filename, target):
 201     """Run a package's installer script against the given target directory."""
 202     log("Unpacking %s..." % filename)
 203     subprocess.check_call(["tar", "xf", filename])
 204     basename = filename.split(".tar")[0]
 205     log("Installing %s..." % basename)
 206     install_cmd = [os.path.join(basename, "install.sh")]
 207     install_cmd += ["--prefix=" + os.path.abspath(target)]
 208     install_cmd += ["--disable-ldconfig"]
 209     subprocess.check_call(install_cmd)
 210     log("Cleaning %s..." % basename)
 211     shutil.rmtree(basename)
 212
 213
 214 def package(manifest, pkg, target):
 215     """Pull out the package dict for a particular package and target
 216     from the given manifest."""
 217     version = manifest["pkg"][pkg]["version"]
 218     if target in manifest["pkg"][pkg]["target"]:
 219         info = manifest["pkg"][pkg]["target"][target]
 220     else:
 221         # rust-src is the same for all targets, and has a literal '*' in the
 222         # section key/name instead of a target
 223         info = manifest["pkg"][pkg]["target"]["*"]
 224     if "xz_url" in info:
 225         info["url"] = info.pop("xz_url")
 226         info["hash"] = info.pop("xz_hash")
 227     return (version, info)
 228
 229
 230 def fetch_package(manifest, pkg, host):
 231     version, info = package(manifest, pkg, host)
 232     if not info["available"]:
 233         log("%s marked unavailable for %s" % (pkg, host))
 234         raise KeyError
 235
 236     log("%s %s\n  %s\n  %s" % (pkg, version, info["url"], info["hash"]))
 237     sha = fetch(info["url"], info["hash"] is not None)
 238     if info["hash"] and sha != info["hash"]:
 239         log(
 240             "Checksum mismatch: package resource is different from manifest"
 241             "\n  %s" % sha
 242         )
 243         raise AssertionError
 244     return info
 245
 246
 247 def fetch_std(manifest, targets):
 248     stds = []
 249     for target in targets:
 250         stds.append(fetch_package(manifest, "rust-std", target))
 251         analysis = fetch_optional(manifest, "rust-analysis", target)
 252         if analysis:
 253             stds.append(analysis)
 254         else:
 255             log(f"Missing rust-analysis for {target}")
 256             # If it's missing for one of the searchfox targets, explicitly
 257             # error out.
 258             if target in (
 259                 "x86_64-unknown-linux-gnu",
 260                 "x86_64-apple-darwin",
 261                 "x86_64-pc-windows-msvc",
 262                 "thumbv7neon-linux-androideabi",
 263             ):
 264                 raise AssertionError
 265
 266     return stds
 267
 268
 269 def fetch_optional(manifest, pkg, host):
 270     try:
 271         return fetch_package(manifest, pkg, host)
 272     except KeyError:
 273         # The package is not available, oh well!
 274         return None
 275
 276
 277 @contextmanager
 278 def chdir(path):
 279     d = os.getcwd()
 280     log('cd "%s"' % path)
 281     os.chdir(path)
 282     try:
 283         yield
 284     finally:
 285         log('cd "%s"' % d)
 286         os.chdir(d)
 287
 288
 289 def build_tar_package(name, base, directory):
 290     name = os.path.realpath(name)
 291     log("tarring {} from {}/{}".format(name, base, directory))
 292     assert name.endswith(".tar.zst")
 293
 294     cctx = zstandard.ZstdCompressor()
 295     with open(name, "wb") as f, cctx.stream_writer(f) as z:
 296         with tarfile.open(mode="w|", fileobj=z) as tf:
 297             with chdir(base):
 298                 tf.add(directory)
 299
 300
 301 def fetch_manifest(channel="stable", host=None, targets=()):
 302     if channel.startswith("bors-"):
 303         assert host
 304         rev = channel[len("bors-") :]
 305         base_url = "https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rustc-builds"
 306         manifest = {
 307             "date": "some date",
 308             "pkg": {},
 309         }
 310
 311         def target(url):
 312             return {
 313                 "url": url,
 314                 "hash": None,
 315                 "available": requests.head(url).status_code == 200,
 316             }
 317
 318         for pkg in (
 319             "cargo",
 320             "rustc",
 321             "rustfmt-preview",
 322             "clippy-preview",
 323             "rust-analyzer-preview",
 324         ):
 325             manifest["pkg"][pkg] = {
 326                 "version": "bors",
 327                 "target": {
 328                     host: target(
 329                         "{}/{}/{}-nightly-{}.tar.xz".format(base_url, rev, pkg, host)
 330                     ),
 331                 },
 332             }
 333         manifest["pkg"]["rust-src"] = {
 334             "version": "bors",
 335             "target": {
 336                 "*": target("{}/{}/rust-src-nightly.tar.xz".format(base_url, rev)),
 337             },
 338         }
 339         for pkg in ("rust-std", "rust-analysis"):
 340             manifest["pkg"][pkg] = {
 341                 "version": "bors",
 342                 "target": {
 343                     t: target(
 344                         "{}/{}/{}-nightly-{}.tar.xz".format(base_url, rev, pkg, t)
 345                     )
 346                     for t in sorted(set(targets) | set([host]))
 347                 },
 348             }
 349         return manifest
 350     if channel.startswith(("beta-", "nightly-")):
 351         channel, date = channel.split("-", 1)
 352         prefix = "/" + date
 353     else:
 354         prefix = ""
 355     url = "https://static.rust-lang.org/dist%s/channel-rust-%s.toml" % (prefix, channel)
 356     req = requests.get(url)
 357     req.raise_for_status()
 358     manifest = toml.loads(req.text)
 359     if manifest["manifest-version"] != "2":
 360         raise NotImplementedError(
 361             "Unrecognized manifest version %s." % manifest["manifest-version"]
 362         )
 363     return manifest
 364
 365
 366 def patch_src(patch, module):
 367     log("Patching Rust src... {} with {}".format(module, patch))
 368     patch = os.path.realpath(patch)
 369     subprocess.check_call(["patch", "-d", module, "-p1", "-i", patch, "--fuzz=0", "-s"])
 370
 371
 372 def build_src(install_dir, host, targets, patches):
 373     install_dir = os.path.abspath(install_dir)
 374     fetches = os.environ["MOZ_FETCHES_DIR"]
 375     rust_dir = os.path.join(fetches, "rust")
 376     patch_dir = os.path.join(os.environ["GECKO_PATH"], "build", "build-rust")
 377
 378     # Clear and remake any previous install directory.
 379     try:
 380         shutil.rmtree(install_dir)
 381     except OSError as e:
 382         if e.errno != errno.ENOENT:
 383             raise
 384     os.makedirs(install_dir)
 385
 386     # Patch the src (see the --patch flag's description for details)
 387     for p in patches:
 388         module, colon, file = p.partition(":")
 389         if not colon:
 390             module, file = "", p
 391         patch_file = os.path.join(patch_dir, file)
 392         patch_module = os.path.join(rust_dir, module)
 393         patch_src(patch_file, patch_module)
 394
 395     log("Building Rust...")
 396
 397     example_config = ""
 398     for example_toml in ("config.example.toml", "config.toml.example"):
 399         path = os.path.join(rust_dir, example_toml)
 400         if os.path.exists(path):
 401             with open(path) as file:
 402                 example_config = file.read()
 403                 break
 404
 405     if "ignore-git" in example_config:
 406         omit_git_hash = "ignore-git"
 407     else:
 408         omit_git_hash = "omit-git-hash"
 409
 410     # Rust builds are configured primarily through a config.toml file.
 411     #
 412     # `sysconfdir` is overloaded to be relative instead of absolute.
 413     # This is the default of `install.sh`, but for whatever reason
 414     # `x.py install` has its own default of `/etc` which we don't want.
 415     base_config = textwrap.dedent(
 416         """
 417         [build]
 418         docs = false
 419         sanitizers = true
 420         profiler = true
 421         extended = true
 422         tools = ["analysis", "cargo", "rustfmt", "clippy", "src", "rust-analyzer"]
 423         cargo-native-static = true
 424
 425         [rust]
 426         {omit_git_hash} = false
 427         use-lld = true
 428
 429         [install]
 430         prefix = "{prefix}"
 431         sysconfdir = "etc"
 432
 433         [llvm]
 434         download-ci-llvm = false
 435         """.format(
 436             prefix=install_dir,
 437             omit_git_hash=omit_git_hash,
 438         )
 439     )
 440
 441     # Rust requires these to be specified per-target
 442     target_config = textwrap.dedent(
 443         """
 444         [target.{target}]
 445         cc = "clang"
 446         cxx = "clang++"
 447         linker = "clang"
 448
 449         """
 450     )
 451
 452     final_config = base_config
 453     for target in sorted(set(targets) | set([host])):
 454         final_config = final_config + target_config.format(target=target)
 455
 456     with open(os.path.join(rust_dir, "config.toml"), "w") as file:
 457         file.write(final_config)
 458
 459     # Setup the env so compilers and toolchains are visible
 460     clang = os.path.join(fetches, "clang")
 461     clang_bin = os.path.join(clang, "bin")
 462     clang_lib = os.path.join(clang, "lib")
 463     sysroot = os.path.join(fetches, "sysroot")
 464
 465     # The rust build doesn't offer much in terms of overriding compiler flags
 466     # when it builds LLVM's compiler-rt, but we want to build with a sysroot.
 467     # So, we create wrappers for clang and clang++ that add the sysroot to the
 468     # command line.
 469     with tempfile.TemporaryDirectory() as tmpdir:
 470         for exe in ("clang", "clang++"):
 471             tmp_exe = os.path.join(tmpdir, exe)
 472             with open(tmp_exe, "w") as fh:
 473                 fh.write("#!/bin/sh\n")
 474                 fh.write(f'exec {clang_bin}/{exe} --sysroot={sysroot} "$@"\n')
 475             os.chmod(tmp_exe, 0o755)
 476
 477         env = os.environ.copy()
 478         env.update(
 479             {
 480                 "PATH": os.pathsep.join((tmpdir, clang_bin, os.environ["PATH"])),
 481                 "LD_LIBRARY_PATH": clang_lib,
 482             }
 483         )
 484
 485         # x.py install does everything we need for us.
 486         # If you're running into issues, consider using `-vv` to debug it.
 487         command = ["python3", "x.py", "install", "-v", "--host", host]
 488         for target in targets:
 489             command.extend(["--target", target])
 490
 491         subprocess.check_call(command, stderr=subprocess.STDOUT, env=env, cwd=rust_dir)
 492
 493
 494 def repack(
 495     host,
 496     targets,
 497     channel="stable",
 498     cargo_channel=None,
 499     patches=[],
 500 ):
 501     install_dir = "rustc"
 502     if channel == "dev":
 503         build_src(install_dir, host, targets, patches)
 504     else:
 505         if patches:
 506             raise ValueError(
 507                 'Patch specified, but channel "%s" is not "dev"!'
 508                 "\nPatches are only for building from source." % channel
 509             )
 510         log("Repacking rust for %s supporting %s..." % (host, targets))
 511         manifest = fetch_manifest(channel, host, targets)
 512         log("Using manifest for rust %s as of %s." % (channel, manifest["date"]))
 513         if cargo_channel == channel:
 514             cargo_manifest = manifest
 515         else:
 516             cargo_manifest = fetch_manifest(cargo_channel, host, targets)
 517             log(
 518                 "Using manifest for cargo %s as of %s."
 519                 % (cargo_channel, cargo_manifest["date"])
 520             )
 521
 522         log("Fetching packages...")
 523         rustc = fetch_package(manifest, "rustc", host)
 524         cargo = fetch_package(cargo_manifest, "cargo", host)
 525         stds = fetch_std(manifest, targets)
 526         rustsrc = fetch_package(manifest, "rust-src", host)
 527         rustfmt = fetch_optional(manifest, "rustfmt-preview", host)
 528         clippy = fetch_optional(manifest, "clippy-preview", host)
 529         rust_analyzer = fetch_optional(manifest, "rust-analyzer-preview", host)
 530
 531         log("Installing packages...")
 532
 533         # Clear any previous install directory.
 534         try:
 535             shutil.rmtree(install_dir)
 536         except OSError as e:
 537             if e.errno != errno.ENOENT:
 538                 raise
 539         install(os.path.basename(rustc["url"]), install_dir)
 540         install(os.path.basename(cargo["url"]), install_dir)
 541         install(os.path.basename(rustsrc["url"]), install_dir)
 542         if rustfmt:
 543             install(os.path.basename(rustfmt["url"]), install_dir)
 544         if clippy:
 545             install(os.path.basename(clippy["url"]), install_dir)
 546         if rust_analyzer:
 547             install(os.path.basename(rust_analyzer["url"]), install_dir)
 548         for std in stds:
 549             install(os.path.basename(std["url"]), install_dir)
 550             pass
 551
 552     log("Creating archive...")
 553     tar_file = install_dir + ".tar.zst"
 554     build_tar_package(tar_file, ".", install_dir)
 555     shutil.rmtree(install_dir)
 556     log("%s is ready." % tar_file)
 557
 558     upload_dir = os.environ.get("UPLOAD_DIR")
 559     if upload_dir:
 560         # Create the upload directory if it doesn't exist.
 561         try:
 562             log("Creating upload directory in %s..." % os.path.abspath(upload_dir))
 563             os.makedirs(upload_dir)
 564         except OSError as e:
 565             if e.errno != errno.EEXIST:
 566                 raise
 567         # Move the tarball to the output directory for upload.
 568         log("Moving %s to the upload directory..." % tar_file)
 569         shutil.move(tar_file, upload_dir)
 570
 571
 572 def expand_platform(name):
 573     """Expand a shortcut name to a full Rust platform string."""
 574     platforms = {
 575         "android": "armv7-linux-androideabi",
 576         "android_x86": "i686-linux-android",
 577         "android_x86-64": "x86_64-linux-android",
 578         "android_aarch64": "aarch64-linux-android",
 579         "linux64": "x86_64-unknown-linux-gnu",
 580         "linux32": "i686-unknown-linux-gnu",
 581         "mac": "x86_64-apple-darwin",
 582         "macos": "x86_64-apple-darwin",
 583         "mac64": "x86_64-apple-darwin",
 584         "mac32": "i686-apple-darwin",
 585         "win64": "x86_64-pc-windows-msvc",
 586         "win32": "i686-pc-windows-msvc",
 587         "mingw32": "i686-pc-windows-gnu",
 588     }
 589     return platforms.get(name, name)
 590
 591
 592 def validate_channel(channel):
 593     """Require a specific release version.
 594
 595     Packaging from meta-channels, like `stable`, `beta`, or `nightly`
 596     doesn't give repeatable output. Reject such channels."""
 597     channel_prefixes = ("stable", "beta", "nightly")
 598     if any([channel.startswith(c) for c in channel_prefixes]):
 599         if "-" not in channel:
 600             raise ValueError(
 601                 'Generic channel "%s" specified!'
 602                 "\nPlease give a specific release version"
 603                 ' like "1.24.0" or "beta-2018-02-20".' % channel
 604             )
 605
 606
 607 def args():
 608     """Read command line arguments and return options."""
 609     parser = argparse.ArgumentParser()
 610     parser.add_argument(
 611         "--channel",
 612         help="Release channel to use:"
 613         " 1.xx.y, beta-yyyy-mm-dd,"
 614         " nightly-yyyy-mm-dd,"
 615         " bors-$rev (grab a build from rust's CI),"
 616         " or dev (build from source).",
 617         required=True,
 618     )
 619     parser.add_argument(
 620         "--allow-generic-channel",
 621         action="store_true",
 622         help='Allow to use e.g. "nightly" without a date as a channel.',
 623     )
 624     parser.add_argument(
 625         "--patch",
 626         dest="patches",
 627         action="append",
 628         default=[],
 629         help="apply the given patch file to a dev build."
 630         " Patch files should be placed in /build/build-rust."
 631         " Patches can be prefixed with `module-path:` to specify they"
 632         " apply to that git submodule in the Rust source."
 633         " e.g. `src/llvm-project:mypatch.diff` patches rust's llvm."
 634         " Can be given more than once.",
 635     )
 636     parser.add_argument(
 637         "--cargo-channel",
 638         help="Release channel version to use for cargo."
 639         " Defaults to the same as --channel.",
 640     )
 641     parser.add_argument(
 642         "--host",
 643         help="Host platform for the toolchain executable:"
 644         " e.g. linux64 or aarch64-linux-android."
 645         " Defaults to linux64.",
 646     )
 647     parser.add_argument(
 648         "--target",
 649         dest="targets",
 650         action="append",
 651         default=[],
 652         help="Additional target platform to support:"
 653         " e.g. linux32 or i686-pc-windows-gnu."
 654         " can be given more than once.",
 655     )
 656     args = parser.parse_args()
 657     if not args.cargo_channel:
 658         args.cargo_channel = args.channel
 659     if not args.allow_generic_channel:
 660         validate_channel(args.channel)
 661         validate_channel(args.cargo_channel)
 662     if not args.host:
 663         args.host = "linux64"
 664     args.host = expand_platform(args.host)
 665     args.targets = [expand_platform(t) for t in args.targets]
 666     delattr(args, "allow_generic_channel")
 667
 668     return args
 669
 670
 671 if __name__ == "__main__":
 672     args = vars(args())
 673     setup_gpg()
 674     repack(**args)