roms/edk2-build.py

   1 #!/usr/bin/python3
   2 """
   3 build helper script for edk2, see
   4 https://gitlab.com/kraxel/edk2-build-config
   5
   6 """
   7 import os
   8 import sys
   9 import shutil
  10 import argparse
  11 import subprocess
  12 import configparser
  13
  14 rebase_prefix    = ""
  15 version_override = None
  16 release_date     = None
  17
  18 # pylint: disable=unused-variable
  19 def check_rebase():
  20     """ detect 'git rebase -x edk2-build.py master' testbuilds """
  21     global rebase_prefix
  22     global version_override
  23     gitdir = '.git'
  24
  25     if os.path.isfile(gitdir):
  26         with open(gitdir, 'r', encoding = 'utf-8') as f:
  27             (unused, gitdir) = f.read().split()
  28
  29     if not os.path.exists(f'{gitdir}/rebase-merge/msgnum'):
  30         return
  31     with open(f'{gitdir}/rebase-merge/msgnum', 'r', encoding = 'utf-8') as f:
  32         msgnum = int(f.read())
  33     with open(f'{gitdir}/rebase-merge/end', 'r', encoding = 'utf-8') as f:
  34         end = int(f.read())
  35     with open(f'{gitdir}/rebase-merge/head-name', 'r', encoding = 'utf-8') as f:
  36         head = f.read().strip().split('/')
  37
  38     rebase_prefix = f'[ {int(msgnum/2)} / {int(end/2)} - {head[-1]} ] '
  39     if msgnum != end and not version_override:
  40         # fixed version speeds up builds
  41         version_override = "test-build-patch-series"
  42
  43 def get_coredir(cfg):
  44     if cfg.has_option('global', 'core'):
  45         return os.path.abspath(cfg['global']['core'])
  46     return os.getcwd()
  47
  48 def get_version(cfg):
  49     coredir = get_coredir(cfg)
  50     if version_override:
  51         version = version_override
  52         print('')
  53         print(f'### version [override]: {version}')
  54         return version
  55     if os.environ.get('RPM_PACKAGE_NAME'):
  56         version = os.environ.get('RPM_PACKAGE_NAME')
  57         version += '-' + os.environ.get('RPM_PACKAGE_VERSION')
  58         version += '-' + os.environ.get('RPM_PACKAGE_RELEASE')
  59         print('')
  60         print(f'### version [rpmbuild]: {version}')
  61         return version
  62     if os.path.exists(coredir + '/.git'):
  63         cmdline = [ 'git', 'describe', '--tags', '--abbrev=8',
  64                     '--match=edk2-stable*' ]
  65         result = subprocess.run(cmdline, cwd = coredir,
  66                                 stdout = subprocess.PIPE,
  67                                 check = True)
  68         version = result.stdout.decode().strip()
  69         print('')
  70         print(f'### version [git]: {version}')
  71         return version
  72     return None
  73
  74 def pcd_string(name, value):
  75     return f'{name}=L{value}\\0'
  76
  77 def pcd_version(cfg):
  78     version = get_version(cfg)
  79     if version is None:
  80         return []
  81     return [ '--pcd', pcd_string('PcdFirmwareVersionString', version) ]
  82
  83 def pcd_release_date():
  84     if release_date is None:
  85         return []
  86     return [ '--pcd', pcd_string('PcdFirmwareReleaseDateString', release_date) ]
  87
  88 def build_message(line, line2 = None):
  89     if os.environ.get('TERM') in [ 'xterm', 'xterm-256color' ]:
  90         # setxterm  title
  91         start  = '\x1b]2;'
  92         end    = '\x07'
  93         print(f'{start}{rebase_prefix}{line}{end}', end = '')
  94
  95     print('')
  96     print('###')
  97     print(f'### {rebase_prefix}{line}')
  98     if line2:
  99         print(f'### {line2}')
 100     print('###', flush = True)
 101
 102 def build_run(cmdline, name, section, silent = False):
 103     print(cmdline, flush = True)
 104     if silent:
 105         print('### building in silent mode ...', flush = True)
 106         result = subprocess.run(cmdline, check = False,
 107                                 stdout = subprocess.PIPE,
 108                                 stderr = subprocess.STDOUT)
 109
 110         logfile = f'{section}.log'
 111         print(f'### writing log to {logfile} ...')
 112         with open(logfile, 'wb') as f:
 113             f.write(result.stdout)
 114
 115         if result.returncode:
 116             print('### BUILD FAILURE')
 117             print('### output')
 118             print(result.stdout.decode())
 119             print(f'### exit code: {result.returncode}')
 120         else:
 121             print('### OK')
 122     else:
 123         result = subprocess.run(cmdline, check = False)
 124     if result.returncode:
 125         print(f'ERROR: {cmdline[0]} exited with {result.returncode}'
 126               f' while building {name}')
 127         sys.exit(result.returncode)
 128
 129 def build_copy(plat, tgt, dstdir, copy):
 130     srcdir = f'Build/{plat}/{tgt}_GCC5'
 131     names = copy.split()
 132     srcfile = names[0]
 133     if len(names) > 1:
 134         dstfile = names[1]
 135     else:
 136         dstfile = os.path.basename(srcfile)
 137     print(f'# copy: {srcdir} / {srcfile}  =>  {dstdir} / {dstfile}')
 138
 139     src = srcdir + '/' + srcfile
 140     dst = dstdir + '/' + dstfile
 141     os.makedirs(os.path.dirname(dst), exist_ok = True)
 142     shutil.copy(src, dst)
 143
 144 def pad_file(dstdir, pad):
 145     args = pad.split()
 146     if len(args) < 2:
 147         raise RuntimeError(f'missing arg for pad ({args})')
 148     name = args[0]
 149     size = args[1]
 150     cmdline = [
 151         'truncate',
 152         '--size', size,
 153         dstdir + '/' + name,
 154     ]
 155     print(f'# padding: {dstdir} / {name}  =>  {size}')
 156     subprocess.run(cmdline, check = True)
 157
 158 # pylint: disable=too-many-branches
 159 def build_one(cfg, build, jobs = None, silent = False):
 160     cmdline  = [ 'build' ]
 161     cmdline += [ '-t', 'GCC5' ]
 162     cmdline += [ '-p', cfg[build]['conf'] ]
 163
 164     if (cfg[build]['conf'].startswith('OvmfPkg/') or
 165         cfg[build]['conf'].startswith('ArmVirtPkg/')):
 166         cmdline += pcd_version(cfg)
 167         cmdline += pcd_release_date()
 168
 169     if jobs:
 170         cmdline += [ '-n', jobs ]
 171     for arch in cfg[build]['arch'].split():
 172         cmdline += [ '-a', arch ]
 173     if 'opts' in cfg[build]:
 174         for name in cfg[build]['opts'].split():
 175             section = 'opts.' + name
 176             for opt in cfg[section]:
 177                 cmdline += [ '-D', opt + '=' + cfg[section][opt] ]
 178     if 'pcds' in cfg[build]:
 179         for name in cfg[build]['pcds'].split():
 180             section = 'pcds.' + name
 181             for pcd in cfg[section]:
 182                 cmdline += [ '--pcd', pcd + '=' + cfg[section][pcd] ]
 183     if 'tgts' in cfg[build]:
 184         tgts = cfg[build]['tgts'].split()
 185     else:
 186         tgts = [ 'DEBUG' ]
 187     for tgt in tgts:
 188         desc = None
 189         if 'desc' in cfg[build]:
 190             desc = cfg[build]['desc']
 191         build_message(f'building: {cfg[build]["conf"]} ({cfg[build]["arch"]}, {tgt})',
 192                       f'description: {desc}')
 193         build_run(cmdline + [ '-b', tgt ],
 194                   cfg[build]['conf'],
 195                   build + '.' + tgt,
 196                   silent)
 197
 198         if 'plat' in cfg[build]:
 199             # copy files
 200             for cpy in cfg[build]:
 201                 if not cpy.startswith('cpy'):
 202                     continue
 203                 build_copy(cfg[build]['plat'],
 204                            tgt,
 205                            cfg[build]['dest'],
 206                            cfg[build][cpy])
 207             # pad builds
 208             for pad in cfg[build]:
 209                 if not pad.startswith('pad'):
 210                     continue
 211                 pad_file(cfg[build]['dest'],
 212                          cfg[build][pad])
 213
 214 def build_basetools(silent = False):
 215     build_message('building: BaseTools')
 216     basedir = os.environ['EDK_TOOLS_PATH']
 217     cmdline = [ 'make', '-C', basedir ]
 218     build_run(cmdline, 'BaseTools', 'build.basetools', silent)
 219
 220 def binary_exists(name):
 221     for pdir in os.environ['PATH'].split(':'):
 222         if os.path.exists(pdir + '/' + name):
 223             return True
 224     return False
 225
 226 def prepare_env(cfg):
 227     """ mimic Conf/BuildEnv.sh """
 228     workspace = os.getcwd()
 229     packages = [ workspace, ]
 230     path = os.environ['PATH'].split(':')
 231     dirs = [
 232         'BaseTools/Bin/Linux-x86_64',
 233         'BaseTools/BinWrappers/PosixLike'
 234     ]
 235
 236     if cfg.has_option('global', 'pkgs'):
 237         for pkgdir in cfg['global']['pkgs'].split():
 238             packages.append(os.path.abspath(pkgdir))
 239     coredir = get_coredir(cfg)
 240     if coredir != workspace:
 241         packages.append(coredir)
 242
 243     # add basetools to path
 244     for pdir in dirs:
 245         p = coredir + '/' + pdir
 246         if not os.path.exists(p):
 247             continue
 248         if p in path:
 249             continue
 250         path.insert(0, p)
 251
 252     # run edksetup if needed
 253     toolsdef = coredir + '/Conf/tools_def.txt'
 254     if not os.path.exists(toolsdef):
 255         os.makedirs(os.path.dirname(toolsdef), exist_ok = True)
 256         build_message('running BaseTools/BuildEnv')
 257         cmdline = [ 'bash', 'BaseTools/BuildEnv' ]
 258         subprocess.run(cmdline, cwd = coredir, check = True)
 259
 260     # set variables
 261     os.environ['PATH'] = ':'.join(path)
 262     os.environ['PACKAGES_PATH'] = ':'.join(packages)
 263     os.environ['WORKSPACE'] = workspace
 264     os.environ['EDK_TOOLS_PATH'] = coredir + '/BaseTools'
 265     os.environ['CONF_PATH'] = coredir + '/Conf'
 266     os.environ['PYTHON_COMMAND'] = '/usr/bin/python3'
 267     os.environ['PYTHONHASHSEED'] = '1'
 268
 269     # for cross builds
 270     if binary_exists('arm-linux-gnu-gcc'):
 271         os.environ['GCC5_ARM_PREFIX'] = 'arm-linux-gnu-'
 272     if binary_exists('loongarch64-linux-gnu-gcc'):
 273         os.environ['GCC5_LOONGARCH64_PREFIX'] = 'loongarch64-linux-gnu-'
 274
 275     hostarch = os.uname().machine
 276     if binary_exists('aarch64-linux-gnu-gcc') and hostarch != 'aarch64':
 277         os.environ['GCC5_AARCH64_PREFIX'] = 'aarch64-linux-gnu-'
 278     if binary_exists('riscv64-linux-gnu-gcc') and hostarch != 'riscv64':
 279         os.environ['GCC5_RISCV64_PREFIX'] = 'riscv64-linux-gnu-'
 280     if binary_exists('x86_64-linux-gnu-gcc') and hostarch != 'x86_64':
 281         os.environ['GCC5_IA32_PREFIX'] = 'x86_64-linux-gnu-'
 282         os.environ['GCC5_X64_PREFIX'] = 'x86_64-linux-gnu-'
 283         os.environ['GCC5_BIN'] = 'x86_64-linux-gnu-'
 284
 285 def build_list(cfg):
 286     for build in cfg.sections():
 287         if not build.startswith('build.'):
 288             continue
 289         name = build.lstrip('build.')
 290         desc = 'no description'
 291         if 'desc' in cfg[build]:
 292             desc = cfg[build]['desc']
 293         print(f'# {name:20s} - {desc}')
 294
 295 def main():
 296     parser = argparse.ArgumentParser(prog = 'edk2-build',
 297                                      description = 'edk2 build helper script')
 298     parser.add_argument('-c', '--config', dest = 'configfile',
 299                         type = str, default = '.edk2.builds', metavar = 'FILE',
 300                         help = 'read configuration from FILE (default: .edk2.builds)')
 301     parser.add_argument('-C', '--directory', dest = 'directory', type = str,
 302                         help = 'change to DIR before building', metavar = 'DIR')
 303     parser.add_argument('-j', '--jobs', dest = 'jobs', type = str,
 304                         help = 'allow up to JOBS parallel build jobs',
 305                         metavar = 'JOBS')
 306     parser.add_argument('-m', '--match', dest = 'match', type = str,
 307                         help = 'only run builds matching INCLUDE (substring)',
 308                         metavar = 'INCLUDE')
 309     parser.add_argument('-x', '--exclude', dest = 'exclude', type = str,
 310                         help = 'skip builds matching EXCLUDE (substring)',
 311                         metavar = 'EXCLUDE')
 312     parser.add_argument('-l', '--list', dest = 'list',
 313                         action = 'store_true', default = False,
 314                         help = 'list build configs available')
 315     parser.add_argument('--silent', dest = 'silent',
 316                         action = 'store_true', default = False,
 317                         help = 'write build output to logfiles, '
 318                         'write to console only on errors')
 319     parser.add_argument('--core', dest = 'core', type = str, metavar = 'DIR',
 320                         help = 'location of the core edk2 repository '
 321                         '(i.e. where BuildTools are located)')
 322     parser.add_argument('--pkg', '--package', dest = 'pkgs',
 323                         type = str, action = 'append', metavar = 'DIR',
 324                         help = 'location(s) of additional packages '
 325                         '(can be specified multiple times)')
 326     parser.add_argument('--version-override', dest = 'version_override',
 327                         type = str, metavar = 'VERSION',
 328                         help = 'set firmware build version')
 329     parser.add_argument('--release-date', dest = 'release_date',
 330                         type = str, metavar = 'DATE',
 331                         help = 'set firmware build release date (in MM/DD/YYYY format)')
 332     options = parser.parse_args()
 333
 334     if options.directory:
 335         os.chdir(options.directory)
 336
 337     if not os.path.exists(options.configfile):
 338         print('config file "{options.configfile}" not found')
 339         return 1
 340
 341     cfg = configparser.ConfigParser()
 342     cfg.optionxform = str
 343     cfg.read(options.configfile)
 344
 345     if options.list:
 346         build_list(cfg)
 347         return
 348
 349     if not cfg.has_section('global'):
 350         cfg.add_section('global')
 351     if options.core:
 352         cfg.set('global', 'core', options.core)
 353     if options.pkgs:
 354         cfg.set('global', 'pkgs', ' '.join(options.pkgs))
 355
 356     global version_override
 357     global release_date
 358     check_rebase()
 359     if options.version_override:
 360         version_override = options.version_override
 361     if options.release_date:
 362         release_date = options.release_date
 363
 364     prepare_env(cfg)
 365     build_basetools(options.silent)
 366     for build in cfg.sections():
 367         if not build.startswith('build.'):
 368             continue
 369         if options.match and options.match not in build:
 370             print(f'# skipping "{build}" (not matching "{options.match}")')
 371             continue
 372         if options.exclude and options.exclude in build:
 373             print(f'# skipping "{build}" (matching "{options.exclude}")')
 374             continue
 375         build_one(cfg, build, options.jobs, options.silent)
 376
 377     return 0
 378
 379 if __name__ == '__main__':
 380     sys.exit(main())