bootstrap/bootstrap.py

   1 #!/usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3
   4 """
   5 bootstrap.py - bootstrapping utility for cabal-install.
   6
   7 See bootstrap/README.md for usage instructions.
   8 """
   9
  10 USAGE = """
  11 This utility is only intended for use in building cabal-install
  12 on a new platform. If you already have a functional (if dated) cabal-install
  13 please rather run `cabal v2-install .`.
  14 """
  15
  16 import argparse
  17 from enum import Enum
  18 import hashlib
  19 import json
  20 from pathlib import Path
  21 import platform
  22 import shutil
  23 import subprocess
  24 import sys
  25 import tempfile
  26 import urllib.request
  27 from textwrap import dedent
  28 from typing import Optional, Dict, List, Tuple, \
  29                    NewType, BinaryIO, NamedTuple
  30
  31 #logging.basicConfig(level=logging.INFO)
  32
  33 BUILDDIR    = Path('_build')
  34
  35 BINDIR      = BUILDDIR / 'bin'            # binaries go there (--bindir)
  36 DISTDIR     = BUILDDIR / 'dists'          # --builddir
  37 UNPACKED    = BUILDDIR / 'unpacked'       # where we unpack tarballs
  38 TARBALLS    = BUILDDIR / 'tarballs'       # where we download tarballks
  39 PSEUDOSTORE = BUILDDIR / 'pseudostore'    # where we install packages
  40 ARTIFACTS   = BUILDDIR / 'artifacts'      # Where we put the archive
  41 TMPDIR      = BUILDDIR / 'tmp'            #
  42 PKG_DB      = BUILDDIR / 'packages.conf'  # package db
  43
  44 PackageName = NewType('PackageName', str)
  45 Version = NewType('Version', str)
  46 SHA256Hash = NewType('SHA256Hash', str)
  47
  48 class PackageSource(Enum):
  49     HACKAGE = 'hackage'
  50     LOCAL = 'local'
  51
  52 BuiltinDep = NamedTuple('BuiltinDep', [
  53     ('package', PackageName),
  54     ('version', Version),
  55 ])
  56
  57 BootstrapDep = NamedTuple('BootstrapDep', [
  58     ('package', PackageName),
  59     ('version', Version),
  60     ('source', PackageSource),
  61     # source tarball SHA256
  62     ('src_sha256', Optional[SHA256Hash]),
  63     # `revision` is only valid when source == HACKAGE.
  64     ('revision', Optional[int]),
  65     ('cabal_sha256', Optional[SHA256Hash]),
  66     ('flags', List[str]),
  67 ])
  68
  69 BootstrapInfo = NamedTuple('BootstrapInfo', [
  70     ('builtin', List[BuiltinDep]),
  71     ('dependencies', List[BootstrapDep]),
  72 ])
  73
  74 FetchInfo = NamedTuple('FetchInfo', [
  75     ('url', str),
  76     ('sha256', SHA256Hash)
  77 ])
  78
  79 FetchPlan = Dict[Path, FetchInfo]
  80
  81 local_packages: List[PackageName] = ["Cabal-syntax", "Cabal", "cabal-install-solver", "cabal-install"]
  82
  83 class Compiler:
  84     def __init__(self, ghc_path: Path):
  85         if not ghc_path.is_file():
  86             raise TypeError(f'GHC {ghc_path} is not a file')
  87
  88         self.ghc_path = ghc_path.resolve()
  89
  90         exe = ''
  91         if platform.system() == 'Windows': exe = '.exe'
  92
  93         info = self._get_ghc_info()
  94         self.version = info['Project version']
  95         #self.lib_dir = Path(info['LibDir'])
  96         #self.ghc_pkg_path = (self.lib_dir / 'bin' / 'ghc-pkg').resolve()
  97         self.ghc_pkg_path = (self.ghc_path.parent / ('ghc-pkg' + exe)).resolve()
  98         if not self.ghc_pkg_path.is_file():
  99             raise TypeError(f'ghc-pkg {self.ghc_pkg_path} is not a file')
 100         self.hsc2hs_path = (self.ghc_path.parent / ('hsc2hs' + exe)).resolve()
 101         if not self.hsc2hs_path.is_file():
 102             raise TypeError(f'hsc2hs {self.hsc2hs_path} is not a file')
 103
 104     def _get_ghc_info(self) -> Dict[str,str]:
 105         from ast import literal_eval
 106         p = subprocess_run([self.ghc_path, '--info'], stdout=subprocess.PIPE, check=True, encoding='UTF-8')
 107         out = p.stdout.replace('\n', '').strip()
 108         return dict(literal_eval(out))
 109
 110 PackageSpec = Tuple[PackageName, Version]
 111
 112 class BadTarball(Exception):
 113     def __init__(self, path: Path, expected_sha256: SHA256Hash, found_sha256: SHA256Hash):
 114         self.path = path
 115         self.expected_sha256 = expected_sha256
 116         self.found_sha256 = found_sha256
 117
 118     def __str__(self):
 119         return '\n'.join([
 120             f'Bad tarball hash: {str(self.path)}',
 121             f'  expected: {self.expected_sha256}',
 122             f'  found:    {self.found_sha256}',
 123         ])
 124
 125 def package_url(package: PackageName, version: Version) -> str:
 126     return f'http://hackage.haskell.org/package/{package}-{version}/{package}-{version}.tar.gz'
 127
 128 def package_cabal_url(package: PackageName, version: Version, revision: int) -> str:
 129     return f'http://hackage.haskell.org/package/{package}-{version}/revision/{revision}.cabal'
 130
 131 def verify_sha256(expected_hash: SHA256Hash, f: Path):
 132     h = hash_file(hashlib.sha256(), f.open('rb'))
 133     if h != expected_hash:
 134         raise BadTarball(f, expected_hash, h)
 135
 136 def read_bootstrap_info(path: Path) -> BootstrapInfo:
 137     obj = json.load(path.open())
 138
 139     def bi_from_json(o: dict) -> BuiltinDep:
 140         return BuiltinDep(**o)
 141
 142     def dep_from_json(o: dict) -> BootstrapDep:
 143         o['source'] = PackageSource(o['source'])
 144         return BootstrapDep(**o)
 145
 146     builtin = [bi_from_json(dep) for dep in obj['builtin'] ]
 147     deps = [dep_from_json(dep) for dep in obj['dependencies'] ]
 148
 149     return BootstrapInfo(dependencies=deps, builtin=builtin)
 150
 151 def check_builtin(dep: BuiltinDep, ghc: Compiler) -> None:
 152     subprocess_run([str(ghc.ghc_pkg_path), 'describe', f'{dep.package}-{dep.version}'],
 153                    check=True, stdout=subprocess.DEVNULL)
 154     print(f'Using {dep.package}-{dep.version} from GHC...')
 155     return
 156
 157 def resolve_dep(dep : BootstrapDep) -> Path:
 158     if dep.source == PackageSource.HACKAGE:
 159
 160         tarball = TARBALLS / f'{dep.package}-{dep.version}.tar.gz'
 161         verify_sha256(dep.src_sha256, tarball)
 162
 163         cabal_file = TARBALLS / f'{dep.package}.cabal'
 164         verify_sha256(dep.cabal_sha256, cabal_file)
 165
 166         UNPACKED.mkdir(parents=True, exist_ok=True)
 167         shutil.unpack_archive(tarball.resolve(), UNPACKED, 'gztar')
 168         sdist_dir = UNPACKED / f'{dep.package}-{dep.version}'
 169
 170         # Update cabal file with revision
 171         if dep.revision is not None:
 172             shutil.copyfile(cabal_file, sdist_dir / f'{dep.package}.cabal')
 173
 174         # We rely on the presence of Setup.hs
 175         if len(list(sdist_dir.glob('Setup.*hs'))) == 0:
 176             with open(sdist_dir / 'Setup.hs', 'w') as f:
 177                 f.write('import Distribution.Simple\n')
 178                 f.write('main = defaultMain\n')
 179
 180     elif dep.source == PackageSource.LOCAL:
 181         if dep.package in local_packages:
 182             sdist_dir = Path(dep.package).resolve()
 183         else:
 184             raise ValueError(f'Unknown local package {dep.package}')
 185     return sdist_dir
 186
 187 def install_dep(dep: BootstrapDep, ghc: Compiler) -> None:
 188     dist_dir = (DISTDIR / f'{dep.package}-{dep.version}').resolve()
 189
 190     sdist_dir = resolve_dep(dep)
 191
 192     install_sdist(dist_dir, sdist_dir, ghc, dep.flags)
 193
 194 def install_sdist(dist_dir: Path, sdist_dir: Path, ghc: Compiler, flags: List[str]):
 195     prefix = PSEUDOSTORE.resolve()
 196     flags_option = ' '.join(flags)
 197     setup_dist_dir = dist_dir / 'setup'
 198     setup = setup_dist_dir / 'Setup'
 199
 200     build_args = [
 201         f'--builddir={dist_dir}',
 202     ]
 203
 204     configure_args = build_args + [
 205         f'--package-db={PKG_DB.resolve()}',
 206         f'--prefix={prefix}',
 207         f'--bindir={BINDIR.resolve()}',
 208         f'--with-compiler={ghc.ghc_path}',
 209         f'--with-hc-pkg={ghc.ghc_pkg_path}',
 210         f'--with-hsc2hs={ghc.hsc2hs_path}',
 211         f'--flags={flags_option}',
 212     ]
 213
 214     def check_call(args: List[str]) -> None:
 215         subprocess_run(args, cwd=sdist_dir, check=True)
 216
 217     setup_dist_dir.mkdir(parents=True, exist_ok=True)
 218
 219     # Note: we pass -i so GHC doesn't look for anything else
 220     # This should be fine for cabal-install dependencies.
 221     check_call([str(ghc.ghc_path), '--make', '-package-env=-', '-i', f'-odir={setup_dist_dir}', f'-hidir={setup_dist_dir}', '-o', setup, 'Setup'])
 222     check_call([setup, 'configure'] + configure_args)
 223     check_call([setup, 'build'] + build_args)
 224     check_call([setup, 'install'] + build_args)
 225
 226 def hash_file(h, f: BinaryIO) -> SHA256Hash:
 227     while True:
 228         d = f.read(1024)
 229         if len(d) == 0:
 230             return SHA256Hash(h.hexdigest())
 231
 232         h.update(d)
 233
 234
 235 # Cabal plan.json representation
 236 UnitId = NewType('UnitId', str)
 237 PlanUnit = NewType('PlanUnit', dict)
 238
 239 def bootstrap(info: BootstrapInfo, ghc: Compiler) -> None:
 240     if not PKG_DB.exists():
 241         print(f'Creating package database {PKG_DB}')
 242         PKG_DB.parent.mkdir(parents=True, exist_ok=True)
 243         subprocess_run([ghc.ghc_pkg_path, 'init', PKG_DB])
 244
 245     for dep in info.builtin:
 246         check_builtin(dep, ghc)
 247
 248     for dep in info.dependencies:
 249         install_dep(dep, ghc)
 250
 251 # Steps
 252 #######################################################################
 253
 254 def linuxname(i, r):
 255   i = i.strip() # id
 256   r = r.strip() # release
 257   if i == '': return 'linux'
 258   else: return f"{i}-{r}".lower()
 259
 260 def macname(macver):
 261   # https://en.wikipedia.org/wiki/MacOS_version_history#Releases
 262   if macver.startswith('10.12.'): return 'sierra'
 263   if macver.startswith('10.13.'): return 'high-sierra'
 264   if macver.startswith('10.14.'): return 'mojave'
 265   if macver.startswith('10.15.'): return 'catalina'
 266   if macver.startswith('11.0.'): return 'big-sur'
 267   else: return macver
 268
 269 def archive_name(cabalversion):
 270     # Ask platform information
 271     machine = platform.machine()
 272     if machine == '': machine = "unknown"
 273
 274     system = platform.system().lower()
 275     if system == '': system = "unknown"
 276
 277     version = system
 278     if system == 'linux':
 279         try:
 280             i = subprocess_run(['lsb_release', '-si'], stdout=subprocess.PIPE, encoding='UTF-8')
 281             r = subprocess_run(['lsb_release', '-sr'], stdout=subprocess.PIPE, encoding='UTF-8')
 282             version = linuxname(i.stdout, r.stdout)
 283         except:
 284             try:
 285                 with open('/etc/alpine-release') as f:
 286                     alpinever = f.read().strip()
 287                     return f'alpine-{alpinever}'
 288             except:
 289                 pass
 290     elif system == 'darwin':
 291         version = 'darwin-' + macname(platform.mac_ver()[0])
 292     elif system == 'freebsd':
 293         version = 'freebsd-' + platform.release().lower()
 294
 295     return f'cabal-install-{cabalversion}-{machine}-{version}'
 296
 297 def make_distribution_archive(cabal_path):
 298     import tempfile
 299
 300     print(f'Creating distribution tarball')
 301
 302     # Get bootstrapped cabal version
 303     # This also acts as smoke test
 304     p = subprocess_run([cabal_path, '--numeric-version'], stdout=subprocess.PIPE, check=True, encoding='UTF-8')
 305     cabalversion = p.stdout.replace('\n', '').strip()
 306
 307     # Archive name
 308     basename = ARTIFACTS.resolve() / (archive_name(cabalversion) + '-bootstrapped')
 309
 310     # In temporary directory, create a directory which we will archive
 311     tmpdir = TMPDIR.resolve()
 312     tmpdir.mkdir(parents=True, exist_ok=True)
 313
 314     rootdir = Path(tempfile.mkdtemp(dir=tmpdir))
 315     shutil.copy(cabal_path, rootdir / 'cabal')
 316
 317     # Make archive...
 318     fmt = 'xztar'
 319     if platform.system() == 'Windows': fmt = 'zip'
 320     archivename = shutil.make_archive(basename, fmt, rootdir)
 321
 322     return archivename
 323
 324 def fetch_from_plan(plan : FetchPlan, output_dir : Path):
 325   output_dir.resolve()
 326   output_dir.mkdir(parents=True, exist_ok=True)
 327
 328   for path in plan:
 329     output_path = output_dir / path
 330     url = plan[path].url
 331     sha = plan[path].sha256
 332     if not output_path.exists():
 333       print(f'Fetching {url}...')
 334       with urllib.request.urlopen(url, timeout = 10) as resp:
 335         shutil.copyfileobj(resp, output_path.open('wb'))
 336     verify_sha256(sha, output_path)
 337
 338 def gen_fetch_plan(info : BootstrapInfo) -> FetchPlan :
 339     sources_dict = {}
 340     for dep in info.dependencies:
 341       if not(dep.package in local_packages):
 342         sources_dict[f"{dep.package}-{dep.version}.tar.gz"] = FetchInfo(package_url(dep.package, dep.version), dep.src_sha256)
 343         if dep.revision is not None:
 344           sources_dict[f"{dep.package}.cabal"] = FetchInfo(package_cabal_url(dep.package, dep.version, dep.revision), dep.cabal_sha256)
 345     return sources_dict
 346
 347 def find_ghc(compiler) -> Compiler:
 348   if compiler is None:
 349       path = shutil.which('ghc')
 350       if path is None:
 351           raise ValueError("Couldn't find ghc in PATH")
 352       ghc = Compiler(Path(path))
 353   else:
 354       ghc = Compiler(compiler)
 355   return ghc
 356
 357 def main() -> None:
 358     parser = argparse.ArgumentParser(
 359         description="bootstrapping utility for cabal-install.",
 360         epilog = USAGE,
 361         formatter_class = argparse.RawDescriptionHelpFormatter)
 362     parser.add_argument('-d', '--deps', type=Path,
 363                         help='bootstrap dependency file')
 364     parser.add_argument('-w', '--with-compiler', type=Path,
 365                         help='path to GHC')
 366     parser.add_argument('-s', '--bootstrap-sources', type=Path,
 367                         help='path to prefetched bootstrap sources archive')
 368     parser.add_argument('--archive', dest='want_archive', action='store_true')
 369     parser.add_argument('--no-archive', dest='want_archive', action='store_false')
 370     parser.set_defaults(want_archive=True)
 371
 372     subparsers = parser.add_subparsers(dest="command")
 373
 374     parser_fetch = subparsers.add_parser('build', help='build cabal-install (default)')
 375
 376     parser_fetch = subparsers.add_parser('fetch', help='fetch all required sources from Hackage (for offline builds)')
 377     parser_fetch.add_argument('-o','--output', type=Path, default='bootstrap-sources')
 378
 379     args = parser.parse_args()
 380
 381     print(dedent("""
 382         DO NOT use this script if you have another recent cabal-install available.
 383         This script is intended only for bootstrapping cabal-install on new
 384         architectures.
 385     """))
 386
 387     ghc = find_ghc(args.with_compiler)
 388
 389     sources_fmt = 'gztar'
 390     if platform.system() == 'Windows': sources_fmt = 'zip'
 391
 392     if args.deps is None:
 393       # We have a tarball with all the required information, unpack it
 394       if args.bootstrap_sources is not None:
 395         print(f'Unpacking {args.bootstrap_sources} to {TARBALLS}')
 396         shutil.unpack_archive(args.bootstrap_sources.resolve(), TARBALLS, sources_fmt)
 397         args.deps = TARBALLS / 'plan-bootstrap.json'
 398         print(f"using plan-bootstrap.json ({args.deps}) from {args.bootstrap_sources}")
 399       else:
 400         print("The bootstrap script requires a bootstrap plan JSON file.")
 401         print("See bootstrap/README.md for more information.")
 402         sys.exit(1)
 403
 404     info = read_bootstrap_info(args.deps)
 405
 406     if args.command == 'fetch':
 407         plan = gen_fetch_plan(info)
 408
 409         print(f'Fetching sources to bootstrap cabal-install with GHC {ghc.version} at {ghc.ghc_path}...')
 410
 411         # In temporary directory, create a directory which we will archive
 412         tmpdir = TMPDIR.resolve()
 413         tmpdir.mkdir(parents=True, exist_ok=True)
 414
 415         rootdir = Path(tempfile.mkdtemp(dir=tmpdir))
 416
 417         fetch_from_plan(plan, rootdir)
 418
 419         shutil.copyfile(args.deps, rootdir / 'plan-bootstrap.json')
 420
 421         archivename = shutil.make_archive(args.output, sources_fmt, root_dir=rootdir)
 422
 423         print(dedent(f"""
 424             Bootstrap sources saved to {archivename}
 425
 426             Use these with the command:
 427
 428             bootstrap.py -w {ghc.ghc_path} -s {archivename}
 429             """))
 430
 431     else: # 'build' command (default behaviour)
 432
 433         print(f'Bootstrapping cabal-install with GHC {ghc.version} at {ghc.ghc_path}...')
 434
 435         if args.bootstrap_sources is None:
 436           plan = gen_fetch_plan(info)
 437           fetch_from_plan(plan, TARBALLS)
 438
 439         bootstrap(info, ghc)
 440         cabal_path = (BINDIR / 'cabal').resolve()
 441
 442         print(dedent(f'''
 443             Bootstrapping finished!
 444
 445             The resulting cabal-install executable can be found at
 446
 447               {cabal_path}
 448             '''))
 449
 450         if args.want_archive:
 451             dist_archive = make_distribution_archive(cabal_path)
 452
 453             print(dedent(f'''
 454                 The cabal-install executable has been archived for distribution in
 455
 456                   {dist_archive}
 457                 '''))
 458
 459         print(dedent(f'''
 460             You now should use this to build a full cabal-install distribution
 461             using v2-build.
 462             '''))
 463
 464 def subprocess_run(args, **kwargs):
 465     "Like subprocess.run, but also print what we run"
 466
 467     args_str = ' '.join(map(str, args))
 468     extras = ''
 469     if 'cwd' in kwargs:
 470         extras += f' cwd={kwargs["cwd"]}'
 471     print(f'bootstrap: running{extras} {args_str}')
 472
 473     return subprocess.run(args, **kwargs)
 474
 475 if __name__ == '__main__':
 476     main()