release.py

   1 #!/usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3
   4 """
   5 release.py - build the release of cabal-install"
   6 """
   7
   8 USAGE = """
   9 This utility is only intended for use in building cabal-install
  10 binary distributions on platforms with existing cabal-install.
  11 """
  12
  13 # TODO, by using v2-install we build from sdists, which is good
  14 # But we cannot get plan.json, to get dependency-receipt
  15 # https://github.com/haskell/cabal/issues/6988
  16
  17 # TODO provide DWARF enabled builds?
  18
  19 # We don't build documentation, its well built by readthedocs.
  20 # We cannot make tarball, as the private key for signing should be on the builder machine.
  21 # We also don't use caching, this way we have one moving part less.
  22
  23 import os
  24 import platform
  25 import shutil
  26 import subprocess
  27
  28 from pathlib import Path
  29 from textwrap import dedent
  30 from typing import NamedTuple
  31
  32 DEFAULT_INDEXSTATE='2020-07-23T11:14:13Z'
  33
  34 Args = NamedTuple('Args', [
  35     ('compiler', Path),
  36     ('cabal', Path),
  37     ('indexstate', str),
  38     ('builddir', Path),
  39     ('static', bool),
  40     ('ofdlocking', bool),
  41     ('tarlib', Path),
  42     ('tarsolver', Path),
  43     ('tarexe', Path),
  44 ])
  45
  46 # utils
  47 #######################################################################
  48
  49 def subprocess_run(args, **kwargs):
  50     "Like subprocess.run, but also print what we run"
  51
  52     args = list(map(str, args)) # For Windows, https://www.scivision.dev/windows-python-pathlib-subprocess-bug/
  53     args_str = ' '.join(map(str, args))
  54     extras = ''
  55     if 'cwd' in kwargs:
  56         extras += f' cwd={kwargs["cwd"]}'
  57     print(f'%{extras} {args_str}')
  58
  59     return subprocess.run(args, **kwargs)
  60
  61 # archive name
  62 #######################################################################
  63
  64 def linuxname(i, r):
  65   i = i.strip() # id
  66   r = r.strip() # release
  67   if i == '': return 'linux'
  68   else: return f"{i}-{r}".lower()
  69
  70 def macname(macver):
  71   # https://en.wikipedia.org/wiki/MacOS_version_history#Releases
  72   if macver.startswith('10.12.'): return 'sierra'
  73   if macver.startswith('10.13.'): return 'high-sierra'
  74   if macver.startswith('10.14.'): return 'mojave'
  75   if macver.startswith('10.15.'): return 'catalina'
  76   if macver.startswith('11.0.'): return 'big-sur'
  77   else: return macver
  78
  79 def archive_name(cabalversion):
  80     # Ask platform information
  81     machine = platform.machine().lower()
  82     if machine == '': machine = "unknown"
  83     if machine == 'amd64': machine = "x86_64"
  84
  85     system = platform.system().lower()
  86     if system == '': system = "unknown"
  87
  88     version = system
  89     if system == 'linux':
  90         try:
  91             i = subprocess_run(['lsb_release', '-si'], stdout=subprocess.PIPE, encoding='UTF-8')
  92             r = subprocess_run(['lsb_release', '-sr'], stdout=subprocess.PIPE, encoding='UTF-8')
  93             version = linuxname(i.stdout, r.stdout)
  94         except:
  95             try:
  96                 with open('/etc/alpine-release') as f:
  97                     alpinever = f.read().strip()
  98                     version = f'alpine-{alpinever}'
  99             except:
 100                 pass
 101     elif system == 'darwin':
 102         version = 'darwin-' + macname(platform.mac_ver()[0])
 103     elif system == 'freebsd':
 104         version = 'freebsd-' + platform.release().lower()
 105
 106     return f'cabal-install-{cabalversion}-{machine}-{version}'
 107
 108 # Steps
 109 #######################################################################
 110
 111 def step_makedirs(args: Args):
 112     (args.builddir / 'bin').mkdir(parents=True, exist_ok=True)
 113     (args.builddir / 'cabal').mkdir(parents=True, exist_ok=True)
 114
 115 def step_config(args: Args):
 116     splitsections = ''
 117     if platform.system() == 'Linux':
 118         splitsections = 'split-sections: True'
 119
 120     # https://github.com/Mistuke/CabalChoco/blob/d0e1d2fd8ce13ab4271c4b906ca0bde3b710a310/3.2.0.0/cabal/tools/chocolateyInstall.ps1#L289
 121     extraprogpath = str(args.builddir / 'bin')
 122     if platform.system() == 'Windows':
 123         msysbin = Path('C:\\tools\\msys64\\usr\\bin')
 124         if msysbin.is_dir():
 125             extraprogpath = extraprogpath + "," + str(msysbin)
 126
 127     # cabal.config
 128     config = dedent(f"""
 129         repository hackage.haskell.org
 130           url: http://hackage.haskell.org/
 131
 132         remote-build-reporting: anonymous
 133         remote-repo-cache:      {args.builddir}/cabal/packages
 134
 135         write-ghc-environment-files: never
 136         install-method: copy
 137         overwrite-policy: always
 138
 139         documentation: False
 140         {splitsections}
 141
 142         build-summary:     {args.builddir}/cabal/logs/build.log
 143         installdir:        {args.builddir}/bin
 144         logs-dir:          {args.builddir}/cabal/logs
 145         store-dir:         {args.builddir}/cabal/store
 146         symlink-bindir:    {args.builddir}/bin
 147         extra-prog-path:   {extraprogpath}
 148
 149         jobs: 1
 150
 151         install-dirs user
 152           prefix: {args.builddir}
 153     """)
 154
 155     with open(args.builddir / 'cabal' / 'config', 'w') as f:
 156         f.write(config)
 157
 158     # cabal.project
 159     cabal_project = dedent(f"""
 160         packages: {args.tarlib}
 161         packages: {args.tarexe}
 162         packages: {args.tarsolver}
 163         tests: False
 164         benchmarks: False
 165         optimization: True
 166
 167         package Cabal
 168           ghc-options: -fexpose-all-unfoldings -fspecialise-aggressively
 169
 170         package parsec
 171           ghc-options: -fexpose-all-unfoldings
 172     """)
 173
 174     if args.static:
 175         # --enable-executable-static doesn't affect "non local" executables, as in v2-install project
 176         cabal_project += dedent("""
 177             package cabal-install
 178                 executable-static: True
 179         """)
 180     cabal_project += dedent(f"""
 181         package lukko
 182             flags: {'+' if args.ofdlocking else '-'}ofd_locking
 183     """)
 184
 185     with open(args.builddir / 'cabal.project', 'w') as f:
 186         f.write(cabal_project)
 187
 188 def make_env(args: Args):
 189     env = {
 190         'PATH': os.environ['PATH'],
 191         'CABAL_DIR': str(args.builddir),
 192         'CABAL_CONFIG': str(args.builddir / 'cabal' / 'config'),
 193     }
 194     # https://superuser.com/questions/1079017/is-there-an-environment-variable-for-c-users-username-appdata-local-temp-in-w
 195     # In particular, we surely need 'TEMP'
 196     # And also SYSTEMROOT to make 'curl' work!
 197     envvars = [
 198         'LANG',
 199         'HOME', 'HOMEDRIVE', 'HOMEPATH',
 200         'TMP', 'TEMP',
 201         'PATHEXT', 'APPDATA', 'LOCALAPPDATA', 'SYSTEMROOT',
 202     ]
 203     for key in envvars:
 204         if key in os.environ:
 205             env[key] = os.environ[key]
 206
 207     return env
 208
 209 def step_cabal_update(args: Args):
 210     env = make_env(args)
 211     subprocess_run([
 212         args.cabal,
 213         'v2-update',
 214         '-v',
 215         f'--index-state={args.indexstate}',
 216     ], cwd=args.builddir, check=True, env=env)
 217
 218 def step_cabal_install(args: Args):
 219     env = make_env(args)
 220     subprocess_run([
 221         args.cabal,
 222         'v2-install',
 223         '-v',
 224         'cabal-install:exe:cabal',
 225         '--project-file=cabal.project',
 226         f'--with-compiler={args.compiler}',
 227     ], cwd=args.builddir, check=True, env=env)
 228
 229 def step_make_archive(args: Args):
 230     import tempfile
 231
 232     print(f'Creating distribution tarball')
 233
 234     # Get bootstrapped cabal version
 235     # This also acts as smoke test
 236     cabal_path = args.builddir / 'bin' / 'cabal'
 237     if platform.system() == 'Windows':
 238         cabal_path = cabal_path.with_suffix('.exe')
 239     p = subprocess_run([cabal_path, '--numeric-version'], stdout=subprocess.PIPE, check=True, encoding='UTF-8')
 240     cabalversion = p.stdout.replace('\n', '').strip()
 241
 242     # Archive name
 243     name = archive_name(cabalversion)
 244     if args.static:
 245         name = name + "-static"
 246     if not args.ofdlocking:
 247         name = name + "-noofd"
 248
 249     basename = args.builddir / 'artifacts' / name
 250
 251     # In temporary directory, create a directory which we will archive
 252     tmpdir = args.builddir / 'tmp'
 253     tmpdir.mkdir(parents=True, exist_ok=True)
 254
 255     rootdir = Path(tempfile.mkdtemp(dir=tmpdir))
 256     shutil.copy(cabal_path, rootdir / 'cabal')
 257
 258     # Make archive...
 259     fmt = 'xztar'
 260     if platform.system() == 'Windows': fmt = 'zip'
 261     archivename = shutil.make_archive(basename, fmt, rootdir)
 262
 263     return archivename
 264
 265 # Main procedure
 266 #######################################################################
 267
 268 def main():
 269     import argparse
 270
 271     parser = argparse.ArgumentParser(
 272         description="release packaging utility for cabal-install.",
 273         epilog = USAGE,
 274         formatter_class = argparse.RawDescriptionHelpFormatter)
 275
 276     class EnableDisable(argparse.Action):
 277         def __call__(self, parser, namespace, values, option_string=None):
 278             value = option_string.startswith('--enable')
 279             setattr(namespace, self.dest, value)
 280
 281     parser.add_argument('-w', '--with-compiler', type=str, default='ghc', help='path to GHC')
 282     parser.add_argument('-C', '--with-cabal', type=str, default='cabal', help='path to cabal-install')
 283     parser.add_argument('-i', '--index-state', type=str, default=DEFAULT_INDEXSTATE, help='index state of Hackage to use')
 284     parser.add_argument('--enable-static-executable', '--disable-static-executable', dest='static', nargs=0, default=False, action=EnableDisable, help='Statically link cabal executable')
 285     parser.add_argument('--enable-ofd-locking', '--disable-ofd-locking', dest='ofd_locking', nargs=0, default=True, action=EnableDisable, help='OFD locking (lukko)')
 286     parser.add_argument('--tarlib', dest='tarlib', required=True, metavar='LIBTAR', help='path to Cabal-version.tar.gz')
 287     parser.add_argument('--tarsolver', dest='tarsolver', required=True, metavar='SOLVERTAR', help='path to cabal-install-solver-version.tar.gz')
 288     parser.add_argument('--tarexe', dest='tarexe', required=True, metavar='EXETAR', help='path to cabal-install-version.tar.gz')
 289     parser.add_argument('--builddir', dest='builddir', type=str, default='_build', help='build directory')
 290
 291     args = parser.parse_args()
 292
 293     args = Args(
 294         compiler   = Path(shutil.which(args.with_compiler)),
 295         cabal      = Path(shutil.which(args.with_cabal)),
 296         indexstate = args.index_state,
 297         builddir   = Path(args.builddir).resolve(),
 298         static     = args.static,
 299         ofdlocking = args.ofd_locking,
 300         tarlib     = Path(args.tarlib).resolve(),
 301         tarexe     = Path(args.tarexe).resolve(),
 302         tarsolver  = Path(args.tarsolver).resolve()
 303     )
 304
 305     print(dedent(f"""
 306         compiler:    {args.compiler}
 307         cabal:       {args.cabal}
 308         index-state: {args.indexstate}
 309         builddir:    {args.builddir}
 310         static:      {args.static}
 311         ofd-locking: {args.ofdlocking}
 312         lib-tarball: {args.tarlib}
 313         solver-tarball: {args.tarsolver}
 314         exe-tarball: {args.tarexe}
 315     """))
 316
 317     # Check tools
 318     subprocess_run([args.compiler, '--version'], check=True)
 319     subprocess_run([args.compiler, '--print-project-git-commit-id'], check=True)
 320     subprocess_run([args.cabal, '--version'], check=True)
 321
 322     step_makedirs(args)
 323     step_config(args)
 324     step_cabal_update(args)
 325     step_cabal_install(args)
 326     archivename = step_make_archive(args)
 327
 328     print(dedent(f'''
 329         Packaging finished!
 330
 331         Distribution have been archived in
 332
 333             {archivename}
 334
 335     '''))
 336
 337 if __name__ == '__main__':
 338     main()