bootstrap/bootstrap.py

   1 #!/usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3
   4 """
   5 bootstrap.py - bootstrapping utility for cabal-install.
   6
   7 See bootstrap/README.md for usage instructions.
   8 """
   9
  10 USAGE = """
  11 This utility is only intended for use in building cabal-install
  12 on a new platform. If you already have a functional (if dated) cabal-install
  13 please rather run `cabal v2-install .`.
  14 """
  15
  16 from enum import Enum
  17 import hashlib
  18 import logging
  19 import json
  20 from pathlib import Path
  21 import platform
  22 import shutil
  23 import subprocess
  24 from textwrap import dedent
  25 from typing import Set, Optional, Dict, List, Tuple, \
  26                    NewType, BinaryIO, NamedTuple, TypeVar
  27
  28 #logging.basicConfig(level=logging.INFO)
  29
  30 BUILDDIR    = Path('_build')
  31
  32 BINDIR      = BUILDDIR / 'bin'            # binaries go there (--bindir)
  33 DISTDIR     = BUILDDIR / 'dists'          # --builddir
  34 UNPACKED    = BUILDDIR / 'unpacked'       # where we unpack tarballs
  35 TARBALLS    = BUILDDIR / 'tarballs'       # where we download tarballks
  36 PSEUDOSTORE = BUILDDIR / 'pseudostore'    # where we install packages
  37 ARTIFACTS   = BUILDDIR / 'artifacts'      # Where we put the archive
  38 TMPDIR      = BUILDDIR / 'tmp'            #
  39 PKG_DB      = BUILDDIR / 'packages.conf'  # package db
  40
  41 PackageName = NewType('PackageName', str)
  42 Version = NewType('Version', str)
  43 SHA256Hash = NewType('SHA256Hash', str)
  44
  45 class PackageSource(Enum):
  46     HACKAGE = 'hackage'
  47     LOCAL = 'local'
  48
  49 BuiltinDep = NamedTuple('BuiltinDep', [
  50     ('package', PackageName),
  51     ('version', Version),
  52 ])
  53
  54 BootstrapDep = NamedTuple('BootstrapDep', [
  55     ('package', PackageName),
  56     ('version', Version),
  57     ('source', PackageSource),
  58     # source tarball SHA256
  59     ('src_sha256', Optional[SHA256Hash]),
  60     # `revision` is only valid when source == HACKAGE.
  61     ('revision', Optional[int]),
  62     ('cabal_sha256', Optional[SHA256Hash]),
  63     ('flags', List[str]),
  64 ])
  65
  66 BootstrapInfo = NamedTuple('BootstrapInfo', [
  67     ('builtin', List[BuiltinDep]),
  68     ('dependencies', List[BootstrapDep]),
  69 ])
  70
  71 class Compiler:
  72     def __init__(self, ghc_path: Path):
  73         if not ghc_path.is_file():
  74             raise TypeError(f'GHC {ghc_path} is not a file')
  75
  76         self.ghc_path = ghc_path.resolve()
  77
  78         info = self._get_ghc_info()
  79         self.version = info['Project version']
  80         #self.lib_dir = Path(info['LibDir'])
  81         #self.ghc_pkg_path = (self.lib_dir / 'bin' / 'ghc-pkg').resolve()
  82         self.ghc_pkg_path = (self.ghc_path.parent / 'ghc-pkg').resolve()
  83         if not self.ghc_pkg_path.is_file():
  84             raise TypeError(f'ghc-pkg {self.ghc_pkg_path} is not a file')
  85         self.hsc2hs_path = (self.ghc_path.parent / 'hsc2hs').resolve()
  86         if not self.hsc2hs_path.is_file():
  87             raise TypeError(f'hsc2hs {self.hsc2hs_path} is not a file')
  88
  89     def _get_ghc_info(self) -> Dict[str,str]:
  90         from ast import literal_eval
  91         p = subprocess_run([self.ghc_path, '--info'], stdout=subprocess.PIPE, check=True, encoding='UTF-8')
  92         out = p.stdout.replace('\n', '').strip()
  93         return dict(literal_eval(out))
  94
  95 PackageSpec = Tuple[PackageName, Version]
  96
  97 class BadTarball(Exception):
  98     def __init__(self, path: Path, expected_sha256: SHA256Hash, found_sha256: SHA256Hash):
  99         self.path = path
 100         self.expected_sha256 = expected_sha256
 101         self.found_sha256 = found_sha256
 102
 103     def __str__(self):
 104         return '\n'.join([
 105             f'Bad tarball hash: {str(self.path)}',
 106             f'  expected: {self.expected_sha256}',
 107             f'  found:    {self.found_sha256}',
 108         ])
 109
 110 def package_url(package: PackageName, version: Version) -> str:
 111     return f'http://hackage.haskell.org/package/{package}-{version}/{package}-{version}.tar.gz'
 112
 113 def package_cabal_url(package: PackageName, version: Version, revision: int) -> str:
 114     return f'http://hackage.haskell.org/package/{package}-{version}/revision/{revision}.cabal'
 115
 116 def verify_sha256(expected_hash: SHA256Hash, f: Path):
 117     h = hash_file(hashlib.sha256(), f.open('rb'))
 118     if h != expected_hash:
 119         raise BadTarball(f, expected_hash, h)
 120
 121 def fetch_package(package: PackageName,
 122                   version: Version,
 123                   src_sha256: SHA256Hash,
 124                   revision: Optional[int],
 125                   cabal_sha256: Optional[SHA256Hash],
 126                   ) -> (Path, Path):
 127     import urllib.request
 128
 129     # Download source distribution
 130     tarball = TARBALLS / f'{package}-{version}.tar.gz'
 131     if not tarball.exists():
 132         print(f'Fetching {package}-{version}...')
 133         tarball.parent.mkdir(parents=True, exist_ok=True)
 134         url = package_url(package, version)
 135         with urllib.request.urlopen(url) as resp:
 136             shutil.copyfileobj(resp, tarball.open('wb'))
 137
 138     verify_sha256(src_sha256, tarball)
 139
 140     # Download revised cabal file
 141     cabal_file = TARBALLS / f'{package}.cabal'
 142     if revision is not None and not cabal_file.exists():
 143         assert cabal_sha256 is not None
 144         url = package_cabal_url(package, version, revision)
 145         with urllib.request.urlopen(url) as resp:
 146             shutil.copyfileobj(resp, cabal_file.open('wb'))
 147             verify_sha256(cabal_sha256, cabal_file)
 148
 149     return (tarball, cabal_file)
 150
 151 def read_bootstrap_info(path: Path) -> BootstrapInfo:
 152     obj = json.load(path.open())
 153
 154     def bi_from_json(o: dict) -> BuiltinDep:
 155         return BuiltinDep(**o)
 156
 157     def dep_from_json(o: dict) -> BootstrapDep:
 158         o['source'] = PackageSource(o['source'])
 159         return BootstrapDep(**o)
 160
 161     builtin = [bi_from_json(dep) for dep in obj['builtin'] ]
 162     deps = [dep_from_json(dep) for dep in obj['dependencies'] ]
 163
 164     return BootstrapInfo(dependencies=deps, builtin=builtin)
 165
 166 def check_builtin(dep: BuiltinDep, ghc: Compiler) -> None:
 167     subprocess_run([str(ghc.ghc_pkg_path), 'describe', f'{dep.package}-{dep.version}'],
 168                    check=True, stdout=subprocess.DEVNULL)
 169     print(f'Using {dep.package}-{dep.version} from GHC...')
 170     return
 171
 172 def install_dep(dep: BootstrapDep, ghc: Compiler) -> None:
 173     dist_dir = (DISTDIR / f'{dep.package}-{dep.version}').resolve()
 174
 175     if dep.source == PackageSource.HACKAGE:
 176         assert dep.src_sha256 is not None
 177         (tarball, cabal_file) = fetch_package(dep.package, dep.version, dep.src_sha256,
 178                                 dep.revision, dep.cabal_sha256)
 179         UNPACKED.mkdir(parents=True, exist_ok=True)
 180         shutil.unpack_archive(tarball.resolve(), UNPACKED, 'gztar')
 181         sdist_dir = UNPACKED / f'{dep.package}-{dep.version}'
 182
 183         # Update cabal file with revision
 184         if dep.revision is not None:
 185             shutil.copyfile(cabal_file, sdist_dir / f'{dep.package}.cabal')
 186
 187     elif dep.source == PackageSource.LOCAL:
 188         if dep.package == 'Cabal':
 189             sdist_dir = Path('Cabal').resolve()
 190         elif dep.package == 'Cabal-syntax':
 191             sdist_dir = Path('Cabal-syntax').resolve()
 192         elif dep.package == 'cabal-install-solver':
 193             sdist_dir = Path('cabal-install-solver').resolve()
 194         elif dep.package == 'cabal-install':
 195             sdist_dir = Path('cabal-install').resolve()
 196         else:
 197             raise ValueError(f'Unknown local package {dep.package}')
 198
 199     install_sdist(dist_dir, sdist_dir, ghc, dep.flags)
 200
 201 def install_sdist(dist_dir: Path, sdist_dir: Path, ghc: Compiler, flags: List[str]):
 202     prefix = PSEUDOSTORE.resolve()
 203     flags_option = ' '.join(flags)
 204     setup_dist_dir = dist_dir / 'setup'
 205     setup = setup_dist_dir / 'Setup'
 206
 207     build_args = [
 208         f'--builddir={dist_dir}',
 209     ]
 210
 211     configure_args = build_args + [
 212         f'--package-db={PKG_DB.resolve()}',
 213         f'--prefix={prefix}',
 214         f'--bindir={BINDIR.resolve()}',
 215         f'--with-compiler={ghc.ghc_path}',
 216         f'--with-hc-pkg={ghc.ghc_pkg_path}',
 217         f'--with-hsc2hs={ghc.hsc2hs_path}',
 218         f'--flags={flags_option}',
 219     ]
 220
 221     def check_call(args: List[str]) -> None:
 222         subprocess_run(args, cwd=sdist_dir, check=True)
 223
 224     setup_dist_dir.mkdir(parents=True, exist_ok=True)
 225
 226     # Note: we pass -i so GHC doesn't look for anything else
 227     # This should be fine for cabal-install dependencies.
 228     check_call([str(ghc.ghc_path), '--make', '-package-env=-', '-i', f'-odir={setup_dist_dir}', f'-hidir={setup_dist_dir}', '-o', setup, 'Setup'])
 229     check_call([setup, 'configure'] + configure_args)
 230     check_call([setup, 'build'] + build_args)
 231     check_call([setup, 'install'] + build_args)
 232
 233 def hash_file(h, f: BinaryIO) -> SHA256Hash:
 234     while True:
 235         d = f.read(1024)
 236         if len(d) == 0:
 237             return SHA256Hash(h.hexdigest())
 238
 239         h.update(d)
 240
 241
 242 # Cabal plan.json representation
 243 UnitId = NewType('UnitId', str)
 244 PlanUnit = NewType('PlanUnit', dict)
 245
 246 def bootstrap(info: BootstrapInfo, ghc: Compiler) -> None:
 247     if not PKG_DB.exists():
 248         print(f'Creating package database {PKG_DB}')
 249         PKG_DB.parent.mkdir(parents=True, exist_ok=True)
 250         subprocess_run([ghc.ghc_pkg_path, 'init', PKG_DB])
 251
 252     for dep in info.builtin:
 253         check_builtin(dep, ghc)
 254
 255     for dep in info.dependencies:
 256         install_dep(dep, ghc)
 257
 258 # Steps
 259 #######################################################################
 260
 261 def linuxname(i, r):
 262   i = i.strip() # id
 263   r = r.strip() # release
 264   if i == '': return 'linux'
 265   else: return f"{i}-{r}".lower()
 266
 267 def macname(macver):
 268   # https://en.wikipedia.org/wiki/MacOS_version_history#Releases
 269   if macver.startswith('10.12.'): return 'sierra'
 270   if macver.startswith('10.13.'): return 'high-sierra'
 271   if macver.startswith('10.14.'): return 'mojave'
 272   if macver.startswith('10.15.'): return 'catalina'
 273   if macver.startswith('11.0.'): return 'big-sur'
 274   else: return macver
 275
 276 def archive_name(cabalversion):
 277     # Ask platform information
 278     machine = platform.machine()
 279     if machine == '': machine = "unknown"
 280
 281     system = platform.system().lower()
 282     if system == '': system = "unknown"
 283
 284     version = system
 285     if system == 'linux':
 286         try:
 287             i = subprocess_run(['lsb_release', '-si'], stdout=subprocess.PIPE, encoding='UTF-8')
 288             r = subprocess_run(['lsb_release', '-sr'], stdout=subprocess.PIPE, encoding='UTF-8')
 289             version = linuxname(i.stdout, r.stdout)
 290         except:
 291             try:
 292                 with open('/etc/alpine-release') as f:
 293                     alpinever = f.read().strip()
 294                     return f'alpine-{alpinever}'
 295             except:
 296                 pass
 297     elif system == 'darwin':
 298         version = 'darwin-' + macname(platform.mac_ver()[0])
 299     elif system == 'freebsd':
 300         version = 'freebsd-' + platform.release().lower()
 301
 302     return f'cabal-install-{cabalversion}-{machine}-{version}'
 303
 304 def make_archive(cabal_path):
 305     import tempfile
 306
 307     print(f'Creating distribution tarball')
 308
 309     # Get bootstrapped cabal version
 310     # This also acts as smoke test
 311     p = subprocess_run([cabal_path, '--numeric-version'], stdout=subprocess.PIPE, check=True, encoding='UTF-8')
 312     cabalversion = p.stdout.replace('\n', '').strip()
 313
 314     # Archive name
 315     basename = ARTIFACTS.resolve() / (archive_name(cabalversion) + '-bootstrapped')
 316
 317     # In temporary directory, create a directory which we will archive
 318     tmpdir = TMPDIR.resolve()
 319     tmpdir.mkdir(parents=True, exist_ok=True)
 320
 321     rootdir = Path(tempfile.mkdtemp(dir=tmpdir))
 322     shutil.copy(cabal_path, rootdir / 'cabal')
 323
 324     # Make archive...
 325     fmt = 'xztar'
 326     if platform.system() == 'Windows': fmt = 'zip'
 327     archivename = shutil.make_archive(basename, fmt, rootdir)
 328
 329     return archivename
 330
 331 def main() -> None:
 332     import argparse
 333     parser = argparse.ArgumentParser(
 334         description="bootstrapping utility for cabal-install.",
 335         epilog = USAGE,
 336         formatter_class = argparse.RawDescriptionHelpFormatter)
 337     parser.add_argument('-d', '--deps', type=Path, default='bootstrap-deps.json',
 338                         help='bootstrap dependency file')
 339     parser.add_argument('-w', '--with-compiler', type=Path,
 340                         help='path to GHC')
 341     args = parser.parse_args()
 342
 343     # Find compiler
 344     if args.with_compiler is None:
 345         path = shutil.which('ghc')
 346         if path is None:
 347             raise ValueError("Couldn't find ghc in PATH")
 348         ghc = Compiler(Path(path))
 349     else:
 350         ghc = Compiler(args.with_compiler)
 351
 352     print(f'Bootstrapping cabal-install with GHC {ghc.version} at {ghc.ghc_path}...')
 353
 354     print(dedent("""
 355         DO NOT use this script if you have another recent cabal-install available.
 356         This script is intended only for bootstrapping cabal-install on new
 357         architectures.
 358     """))
 359
 360     info = read_bootstrap_info(args.deps)
 361     bootstrap(info, ghc)
 362     cabal_path = (BINDIR / 'cabal').resolve()
 363
 364     archive = make_archive(cabal_path)
 365
 366     print(dedent(f'''
 367         Bootstrapping finished!
 368
 369         The resulting cabal-install executable can be found at
 370
 371             {cabal_path}
 372
 373         It have been archived for distribution in
 374
 375             {archive}
 376
 377         You now should use this to build a full cabal-install distribution
 378         using v2-build.
 379     '''))
 380
 381 def subprocess_run(args, **kwargs):
 382     "Like subprocess.run, but also print what we run"
 383
 384     args_str = ' '.join(map(str, args))
 385     extras = ''
 386     if 'cwd' in kwargs:
 387         extras += f' cwd={kwargs["cwd"]}'
 388     print(f'bootstrap: running{extras} {args_str}')
 389
 390     return subprocess.run(args, **kwargs)
 391
 392 if __name__ == '__main__':
 393     main()