bootstrap/bootstrap.py

   1 #!/usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3
   4 """
   5 bootstrap.py - bootstrapping utility for cabal-install.
   6
   7 See bootstrap/README.md for usage instructions.
   8 """
   9
  10 USAGE = """
  11 This utility is only intended for use in building cabal-install
  12 on a new platform. If you already have a functional (if dated) cabal-install
  13 please rather run `cabal install .`.
  14 """
  15
  16 import argparse
  17 from enum import Enum
  18 import hashlib
  19 import json
  20 from pathlib import Path
  21 import platform
  22 import shutil
  23 import subprocess
  24 import sys
  25 import tempfile
  26 import urllib.request
  27 from textwrap import dedent
  28 from typing import Optional, Dict, List, Tuple, \
  29                    NewType, BinaryIO, NamedTuple
  30
  31 #logging.basicConfig(level=logging.INFO)
  32
  33 BUILDDIR    = Path('_build')
  34
  35 BINDIR      = BUILDDIR / 'bin'            # binaries go there (--bindir)
  36 DISTDIR     = BUILDDIR / 'dists'          # --builddir
  37 UNPACKED    = BUILDDIR / 'unpacked'       # where we unpack tarballs
  38 TARBALLS    = BUILDDIR / 'tarballs'       # where we download tarballks
  39 PSEUDOSTORE = BUILDDIR / 'pseudostore'    # where we install packages
  40 ARTIFACTS   = BUILDDIR / 'artifacts'      # Where we put the archive
  41 TMPDIR      = BUILDDIR / 'tmp'            #
  42 PKG_DB      = BUILDDIR / 'packages.conf'  # package db
  43
  44 PackageName = NewType('PackageName', str)
  45 Version = NewType('Version', str)
  46 SHA256Hash = NewType('SHA256Hash', str)
  47
  48 class PackageSource(Enum):
  49     HACKAGE = 'hackage'
  50     LOCAL = 'local'
  51
  52 BuiltinDep = NamedTuple('BuiltinDep', [
  53     ('package', PackageName),
  54     ('version', Version),
  55 ])
  56
  57 BootstrapDep = NamedTuple('BootstrapDep', [
  58     ('package', PackageName),
  59     ('version', Version),
  60     ('source', PackageSource),
  61     # source tarball SHA256
  62     ('src_sha256', Optional[SHA256Hash]),
  63     # `revision` is only valid when source == HACKAGE.
  64     ('revision', Optional[int]),
  65     ('cabal_sha256', Optional[SHA256Hash]),
  66     ('flags', List[str]),
  67     ('component', Optional[str])
  68 ])
  69
  70 BootstrapInfo = NamedTuple('BootstrapInfo', [
  71     ('builtin', List[BuiltinDep]),
  72     ('dependencies', List[BootstrapDep]),
  73 ])
  74
  75 FetchInfo = NamedTuple('FetchInfo', [
  76     ('url', str),
  77     ('sha256', SHA256Hash)
  78 ])
  79
  80 FetchPlan = Dict[Path, FetchInfo]
  81
  82 local_packages: List[PackageName] = ["Cabal-syntax", "Cabal", "cabal-install-solver", "cabal-install"]
  83
  84 class Compiler:
  85     def __init__(self, ghc_path: Path):
  86         if not ghc_path.is_file():
  87             raise TypeError(f'GHC {ghc_path} is not a file')
  88
  89         self.ghc_path = ghc_path.resolve()
  90
  91         exe = ''
  92         if platform.system() == 'Windows': exe = '.exe'
  93
  94         info = self._get_ghc_info()
  95         self.version = info['Project version']
  96         #self.lib_dir = Path(info['LibDir'])
  97         #self.ghc_pkg_path = (self.lib_dir / 'bin' / 'ghc-pkg').resolve()
  98         self.ghc_pkg_path = (self.ghc_path.parent / ('ghc-pkg' + exe)).resolve()
  99         if not self.ghc_pkg_path.is_file():
 100             raise TypeError(f'ghc-pkg {self.ghc_pkg_path} is not a file')
 101         self.hsc2hs_path = (self.ghc_path.parent / ('hsc2hs' + exe)).resolve()
 102         if not self.hsc2hs_path.is_file():
 103             raise TypeError(f'hsc2hs {self.hsc2hs_path} is not a file')
 104
 105     def _get_ghc_info(self) -> Dict[str,str]:
 106         from ast import literal_eval
 107         p = subprocess_run([self.ghc_path, '--info'], stdout=subprocess.PIPE, check=True, encoding='UTF-8')
 108         out = p.stdout.replace('\n', '').strip()
 109         return dict(literal_eval(out))
 110
 111 PackageSpec = Tuple[PackageName, Version]
 112
 113 class BadTarball(Exception):
 114     def __init__(self, path: Path, expected_sha256: SHA256Hash, found_sha256: SHA256Hash):
 115         self.path = path
 116         self.expected_sha256 = expected_sha256
 117         self.found_sha256 = found_sha256
 118
 119     def __str__(self):
 120         return '\n'.join([
 121             f'Bad tarball hash: {str(self.path)}',
 122             f'  expected: {self.expected_sha256}',
 123             f'  found:    {self.found_sha256}',
 124         ])
 125
 126 def package_url(package: PackageName, version: Version) -> str:
 127     return f'http://hackage.haskell.org/package/{package}-{version}/{package}-{version}.tar.gz'
 128
 129 def package_cabal_url(package: PackageName, version: Version, revision: int) -> str:
 130     return f'http://hackage.haskell.org/package/{package}-{version}/revision/{revision}.cabal'
 131
 132 def verify_sha256(expected_hash: SHA256Hash, f: Path):
 133     h = hash_file(hashlib.sha256(), f.open('rb'))
 134     if h != expected_hash:
 135         raise BadTarball(f, expected_hash, h)
 136
 137 def read_bootstrap_info(path: Path) -> BootstrapInfo:
 138     obj = json.load(path.open())
 139
 140     def bi_from_json(o: dict) -> BuiltinDep:
 141         return BuiltinDep(**o)
 142
 143     def dep_from_json(o: dict) -> BootstrapDep:
 144         o['source'] = PackageSource(o['source'])
 145         return BootstrapDep(**o)
 146
 147     builtin = [bi_from_json(dep) for dep in obj['builtin'] ]
 148     deps = [dep_from_json(dep) for dep in obj['dependencies'] ]
 149
 150     return BootstrapInfo(dependencies=deps, builtin=builtin)
 151
 152 def check_builtin(dep: BuiltinDep, ghc: Compiler) -> None:
 153     subprocess_run([str(ghc.ghc_pkg_path), 'describe', f'{dep.package}-{dep.version}'],
 154                    check=True, stdout=subprocess.DEVNULL)
 155     print(f'Using {dep.package}-{dep.version} from GHC...')
 156     return
 157
 158 def resolve_dep(dep : BootstrapDep) -> Path:
 159     if dep.source == PackageSource.HACKAGE:
 160
 161         tarball = TARBALLS / f'{dep.package}-{dep.version}.tar.gz'
 162         verify_sha256(dep.src_sha256, tarball)
 163
 164         cabal_file = TARBALLS / f'{dep.package}.cabal'
 165         verify_sha256(dep.cabal_sha256, cabal_file)
 166
 167         UNPACKED.mkdir(parents=True, exist_ok=True)
 168         shutil.unpack_archive(tarball.resolve(), UNPACKED, 'gztar')
 169         sdist_dir = UNPACKED / f'{dep.package}-{dep.version}'
 170
 171         # Update cabal file with revision
 172         if dep.revision is not None:
 173             shutil.copyfile(cabal_file, sdist_dir / f'{dep.package}.cabal')
 174
 175         # We rely on the presence of Setup.hs
 176         if len(list(sdist_dir.glob('Setup.*hs'))) == 0:
 177             with open(sdist_dir / 'Setup.hs', 'w') as f:
 178                 f.write('import Distribution.Simple\n')
 179                 f.write('main = defaultMain\n')
 180
 181     elif dep.source == PackageSource.LOCAL:
 182         if dep.package in local_packages:
 183             sdist_dir = Path(dep.package).resolve()
 184         else:
 185             raise ValueError(f'Unknown local package {dep.package}')
 186     return sdist_dir
 187
 188 def install_dep(dep: BootstrapDep, ghc: Compiler) -> None:
 189     dist_dir = (DISTDIR / f'{dep.package}-{dep.version}').resolve()
 190
 191     sdist_dir = resolve_dep(dep)
 192
 193     install_sdist(dist_dir, sdist_dir, ghc, dep.flags, dep.component)
 194
 195 def install_sdist(dist_dir: Path, sdist_dir: Path, ghc: Compiler, flags: List[str], component):
 196     prefix = PSEUDOSTORE.resolve()
 197     flags_option = ' '.join(flags)
 198     setup_dist_dir = dist_dir / 'setup'
 199     setup = setup_dist_dir / 'Setup'
 200
 201     build_args = [
 202         f'--builddir={dist_dir}',
 203     ]
 204
 205     configure_args = build_args + [
 206         f'--package-db={PKG_DB.resolve()}',
 207         f'--prefix={prefix}',
 208         f'--bindir={BINDIR.resolve()}',
 209         f'--extra-prog-path={BINDIR.resolve()}',
 210         f'--with-compiler={ghc.ghc_path}',
 211         f'--with-hc-pkg={ghc.ghc_pkg_path}',
 212         f'--with-hsc2hs={ghc.hsc2hs_path}',
 213         f'--flags={flags_option}',
 214         f'{component or ""}'
 215     ]
 216
 217     def check_call(args: List[str]) -> None:
 218         subprocess_run(args, cwd=sdist_dir, check=True)
 219
 220     setup_dist_dir.mkdir(parents=True, exist_ok=True)
 221
 222     # Note: we pass -i so GHC doesn't look for anything else
 223     # This should be fine for cabal-install dependencies.
 224     check_call([str(ghc.ghc_path), '--make', '-package-env=-', '-i', f'-odir={setup_dist_dir}', f'-hidir={setup_dist_dir}', '-o', setup, 'Setup'])
 225     check_call([setup, 'configure'] + configure_args)
 226     check_call([setup, 'build'] + build_args)
 227     check_call([setup, 'install'] + build_args)
 228
 229 def hash_file(h, f: BinaryIO) -> SHA256Hash:
 230     while True:
 231         d = f.read(1024)
 232         if len(d) == 0:
 233             return SHA256Hash(h.hexdigest())
 234
 235         h.update(d)
 236
 237
 238 # Cabal plan.json representation
 239 UnitId = NewType('UnitId', str)
 240 PlanUnit = NewType('PlanUnit', dict)
 241
 242 def bootstrap(info: BootstrapInfo, ghc: Compiler) -> None:
 243     if not PKG_DB.exists():
 244         print(f'Creating package database {PKG_DB}')
 245         PKG_DB.parent.mkdir(parents=True, exist_ok=True)
 246         subprocess_run([ghc.ghc_pkg_path, 'init', PKG_DB])
 247
 248     for dep in info.builtin:
 249         check_builtin(dep, ghc)
 250
 251     for dep in info.dependencies:
 252         install_dep(dep, ghc)
 253
 254 # Steps
 255 #######################################################################
 256
 257 def linuxname(i, r):
 258   i = i.strip() # id
 259   r = r.strip() # release
 260   if i == '': return 'linux'
 261   else: return f"{i}-{r}".lower()
 262
 263 def macname(macver):
 264   # https://en.wikipedia.org/wiki/MacOS_version_history#Releases
 265   if macver.startswith('10.12.'): return 'sierra'
 266   if macver.startswith('10.13.'): return 'high-sierra'
 267   if macver.startswith('10.14.'): return 'mojave'
 268   if macver.startswith('10.15.'): return 'catalina'
 269   if macver.startswith('11.0.'): return 'big-sur'
 270   else: return macver
 271
 272 def archive_name(cabalversion):
 273     # Ask platform information
 274     machine = platform.machine()
 275     if machine == '': machine = "unknown"
 276
 277     system = platform.system().lower()
 278     if system == '': system = "unknown"
 279
 280     version = system
 281     if system == 'linux':
 282         try:
 283             i = subprocess_run(['lsb_release', '-si'], stdout=subprocess.PIPE, encoding='UTF-8')
 284             r = subprocess_run(['lsb_release', '-sr'], stdout=subprocess.PIPE, encoding='UTF-8')
 285             version = linuxname(i.stdout, r.stdout)
 286         except:
 287             try:
 288                 with open('/etc/alpine-release') as f:
 289                     alpinever = f.read().strip()
 290                     return f'alpine-{alpinever}'
 291             except:
 292                 pass
 293     elif system == 'darwin':
 294         version = 'darwin-' + macname(platform.mac_ver()[0])
 295     elif system == 'freebsd':
 296         version = 'freebsd-' + platform.release().lower()
 297
 298     return f'cabal-install-{cabalversion}-{machine}-{version}'
 299
 300 def make_distribution_archive(cabal_path):
 301     import tempfile
 302
 303     print(f'Creating distribution tarball')
 304
 305     # Get bootstrapped cabal version
 306     # This also acts as smoke test
 307     p = subprocess_run([cabal_path, '--numeric-version'], stdout=subprocess.PIPE, check=True, encoding='UTF-8')
 308     cabalversion = p.stdout.replace('\n', '').strip()
 309
 310     # Archive name
 311     basename = ARTIFACTS.resolve() / (archive_name(cabalversion) + '-bootstrapped')
 312
 313     # In temporary directory, create a directory which we will archive
 314     tmpdir = TMPDIR.resolve()
 315     tmpdir.mkdir(parents=True, exist_ok=True)
 316
 317     rootdir = Path(tempfile.mkdtemp(dir=tmpdir))
 318     shutil.copy(cabal_path, rootdir / 'cabal')
 319
 320     # Make archive...
 321     fmt = 'xztar'
 322     if platform.system() == 'Windows': fmt = 'zip'
 323     archivename = shutil.make_archive(basename, fmt, rootdir)
 324
 325     return archivename
 326
 327 def fetch_from_plan(plan : FetchPlan, output_dir : Path):
 328   output_dir.resolve()
 329   output_dir.mkdir(parents=True, exist_ok=True)
 330
 331   for path in plan:
 332     output_path = output_dir / path
 333     url = plan[path].url
 334     sha = plan[path].sha256
 335     if not output_path.exists():
 336       print(f'Fetching {url}...')
 337       with urllib.request.urlopen(url, timeout = 10) as resp:
 338         shutil.copyfileobj(resp, output_path.open('wb'))
 339     verify_sha256(sha, output_path)
 340
 341 def gen_fetch_plan(info : BootstrapInfo) -> FetchPlan :
 342     sources_dict = {}
 343     for dep in info.dependencies:
 344       if not(dep.package in local_packages):
 345         sources_dict[f"{dep.package}-{dep.version}.tar.gz"] = FetchInfo(package_url(dep.package, dep.version), dep.src_sha256)
 346         if dep.revision is not None:
 347           sources_dict[f"{dep.package}.cabal"] = FetchInfo(package_cabal_url(dep.package, dep.version, dep.revision), dep.cabal_sha256)
 348     return sources_dict
 349
 350 def find_ghc(compiler) -> Compiler:
 351   if compiler is None:
 352       path = shutil.which('ghc')
 353       if path is None:
 354           raise ValueError("Couldn't find ghc in PATH")
 355       ghc = Compiler(Path(path))
 356   else:
 357       ghc = Compiler(compiler)
 358   return ghc
 359
 360 def main() -> None:
 361     parser = argparse.ArgumentParser(
 362         description="bootstrapping utility for cabal-install.",
 363         epilog = USAGE,
 364         formatter_class = argparse.RawDescriptionHelpFormatter)
 365     parser.add_argument('-d', '--deps', type=Path,
 366                         help='bootstrap dependency file')
 367     parser.add_argument('-w', '--with-compiler', type=Path,
 368                         help='path to GHC')
 369     parser.add_argument('-s', '--bootstrap-sources', type=Path,
 370                         help='path to prefetched bootstrap sources archive')
 371     parser.add_argument('--archive', dest='want_archive', action='store_true')
 372     parser.add_argument('--no-archive', dest='want_archive', action='store_false')
 373     parser.set_defaults(want_archive=True)
 374
 375     subparsers = parser.add_subparsers(dest="command")
 376
 377     parser_fetch = subparsers.add_parser('build', help='build cabal-install (default)')
 378
 379     parser_fetch = subparsers.add_parser('fetch', help='fetch all required sources from Hackage (for offline builds)')
 380     parser_fetch.add_argument('-o','--output', type=Path, default='bootstrap-sources')
 381
 382     args = parser.parse_args()
 383
 384     print(dedent("""
 385         DO NOT use this script if you have another recent cabal-install available.
 386         This script is intended only for bootstrapping cabal-install on new
 387         architectures.
 388     """))
 389
 390     ghc = find_ghc(args.with_compiler)
 391
 392     sources_fmt = 'gztar'
 393     if platform.system() == 'Windows': sources_fmt = 'zip'
 394
 395     if args.deps is None:
 396       # We have a tarball with all the required information, unpack it
 397       if args.bootstrap_sources is not None:
 398         print(f'Unpacking {args.bootstrap_sources} to {TARBALLS}')
 399         shutil.unpack_archive(args.bootstrap_sources.resolve(), TARBALLS, sources_fmt)
 400         args.deps = TARBALLS / 'plan-bootstrap.json'
 401         print(f"using plan-bootstrap.json ({args.deps}) from {args.bootstrap_sources}")
 402       else:
 403         print("The bootstrap script requires a bootstrap plan JSON file.")
 404         print("See bootstrap/README.md for more information.")
 405         sys.exit(1)
 406
 407     info = read_bootstrap_info(args.deps)
 408
 409     if args.command == 'fetch':
 410         plan = gen_fetch_plan(info)
 411
 412         print(f'Fetching sources to bootstrap cabal-install with GHC {ghc.version} at {ghc.ghc_path}...')
 413
 414         # In temporary directory, create a directory which we will archive
 415         tmpdir = TMPDIR.resolve()
 416         tmpdir.mkdir(parents=True, exist_ok=True)
 417
 418         rootdir = Path(tempfile.mkdtemp(dir=tmpdir))
 419
 420         fetch_from_plan(plan, rootdir)
 421
 422         shutil.copyfile(args.deps, rootdir / 'plan-bootstrap.json')
 423
 424         archivename = shutil.make_archive(args.output, sources_fmt, root_dir=rootdir)
 425
 426         print(dedent(f"""
 427             Bootstrap sources saved to {archivename}
 428
 429             Use these with the command:
 430
 431             bootstrap.py -w {ghc.ghc_path} -s {archivename}
 432             """))
 433
 434     else: # 'build' command (default behaviour)
 435
 436         print(f'Bootstrapping cabal-install with GHC {ghc.version} at {ghc.ghc_path}...')
 437
 438         if args.bootstrap_sources is None:
 439           plan = gen_fetch_plan(info)
 440           fetch_from_plan(plan, TARBALLS)
 441
 442         bootstrap(info, ghc)
 443         cabal_path = (BINDIR / 'cabal').resolve()
 444
 445         print(dedent(f'''
 446             Bootstrapping finished!
 447
 448             The resulting cabal-install executable can be found at
 449
 450               {cabal_path}
 451             '''))
 452
 453         if args.want_archive:
 454             dist_archive = make_distribution_archive(cabal_path)
 455
 456             print(dedent(f'''
 457                 The cabal-install executable has been archived for distribution in
 458
 459                   {dist_archive}
 460                 '''))
 461
 462         print(dedent(f'''
 463             You now should use this to build a full cabal-install distribution
 464             using 'cabal build'.
 465             '''))
 466
 467 def subprocess_run(args, **kwargs):
 468     "Like subprocess.run, but also print what we run"
 469
 470     args_str = ' '.join(map(str, args))
 471     extras = ''
 472     if 'cwd' in kwargs:
 473         extras += f' cwd={kwargs["cwd"]}'
 474     print(f'bootstrap: running{extras} {args_str}')
 475
 476     return subprocess.run(args, **kwargs)
 477
 478 if __name__ == '__main__':
 479     main()