build.py

   1 #!/usr/bin/env python3
   2
   3 #
   4 # Copyright (c) 2017 Vojtech Horky
   5 # All rights reserved.
   6 #
   7 # Redistribution and use in source and binary forms, with or without
   8 # modification, are permitted provided that the following conditions
   9 # are met:
  10 #
  11 # - Redistributions of source code must retain the above copyright
  12 #   notice, this list of conditions and the following disclaimer.
  13 # - Redistributions in binary form must reproduce the above copyright
  14 #   notice, this list of conditions and the following disclaimer in the
  15 #   documentation and/or other materials provided with the distribution.
  16 # - The name of the author may not be used to endorse or promote products
  17 #   derived from this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  20 # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  21 # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  22 # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  23 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  24 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  28 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29 #
  30
  31 import subprocess
  32 import os
  33 import time
  34 import sys
  35 import argparse
  36 import multiprocessing
  37 from hbuild.scheduler import BuildScheduler
  38 from hbuild.cvs import *
  39 from hbuild.builders.helenos import *
  40 from hbuild.builders.coastline import *
  41 from hbuild.builders.tests import *
  42 from hbuild.web import *
  43 from hbuild.output import ConsolePrinter
  44
  45 def create_checkout_task(name, url):
  46     if url.startswith("wip://"):
  47         return RsyncCheckoutTask(name, url[6:])
  48     else:
  49         return BzrCheckoutTask(name, url)
  50
  51 # Command-line options
  52 args = argparse.ArgumentParser(description='HelenOS integration build')
  53 args.add_argument('--helenos-repository', default='bzr://helenos.org/mainline', dest='helenos_repository',
  54     metavar='PATH',
  55     help='HelenOS repository path'
  56 )
  57 args.add_argument('--coastline-repository', default='bzr://helenos.org/coastline', dest='coastline_repository',
  58     metavar='PATH',
  59     help='Coastline repository path'
  60 )
  61 args.add_argument('--build-id', default='0', dest='build_id',
  62     metavar='ID',
  63     help='Build id (typically sequence number).',
  64 )
  65 args.add_argument('--rss-url', default='', dest='rss_url',
  66     metavar='URL_WITH_RSS',
  67     help='URL of RSS for latest builds.'
  68 )
  69 args.add_argument('--resource-path', default=None, dest='web_resource_path',
  70     metavar='RELATIVE_PATH',
  71     help='Path where static web resources are stored (when specified, the resources are NOT copied).'
  72 )
  73 args.add_argument('--build-directory', default=os.path.abspath('tmp'), dest='build_directory',
  74     metavar='DIR',
  75     help='Where to build (space for temporary files).',
  76 )
  77 args.add_argument('--artefact-directory', default=os.path.abspath('out/'), dest='artefact_directory',
  78     metavar='DIR',
  79     help='Where to place downloadable files and HTML report.'
  80 )
  81 args.add_argument('--platforms', default='ALL', dest='platforms',
  82     metavar='PLATFORM1[,PLATFORM2[,...]',
  83     help='Which platforms to build (defaults to all detected ones; can be either machine specific "ia64/ski" or architecture specific "ia64/*").'
  84 )
  85 args.add_argument('--harbours', default='ALL', dest='harbours',
  86     metavar='HARBOUR1[,HARBOUR2[,...]',
  87     help='Which harbours to build (defaults to all detected ones).'
  88 )
  89 args.add_argument('--tests', default='ALL', dest='tests',
  90     metavar='TEST1[,TEST2[,...]]',
  91     help='Which tests to run (shell wildcards supported).')
  92 args.add_argument('--jobs', default=multiprocessing.cpu_count(), dest='jobs',
  93     type=int,
  94     metavar='COUNT',
  95     help='Number of concurrent jobs.'
  96 )
  97 args.add_argument('--no-colors', default=False, dest='no_colors',
  98     action='store_true',
  99     help='Disable colorful output'
 100 )
 101 args.add_argument('--debug', default=False, dest='debug',
 102     action='store_true',
 103     help='Print debugging messages'
 104 )
 105
 106 config = args.parse_args()
 107 config.artefact_directory = os.path.abspath(config.artefact_directory)
 108 config.build_directory = os.path.abspath(config.build_directory)
 109 config.self_path = os.path.dirname(os.path.realpath(sys.argv[0]))
 110
 111 printer = ConsolePrinter(config.no_colors)
 112
 113 scheduler = BuildScheduler(
 114     max_workers=config.jobs,
 115     build=config.build_directory,
 116     artefact=config.artefact_directory,
 117     build_id=config.build_id,
 118     printer=printer,
 119     debug=config.debug
 120 )
 121
 122 #
 123 # Check-out both HelenOS and coastline repositories
 124 scheduler.submit("Checking-out HelenOS",
 125     "helenos-checkout",
 126     create_checkout_task("helenos", config.helenos_repository))
 127 scheduler.submit("Checking-out Coastline",
 128     "coastline-checkout",
 129     create_checkout_task("coastline", config.coastline_repository))
 130
 131 #
 132 # HelenOS (mainline): get list of profiles (i.e. supported architectures
 133 # and platforms) and build all of them
 134 scheduler.submit("Determininig available profiles",
 135     "helenos-get-profiles",
 136     HelenOSGetProfilesTask(config.platforms.split(',')),
 137     ["helenos-checkout"])
 138
 139
 140 scheduler.submit("Schedule HelenOS builds",
 141     "helenos-build",
 142     HelenOSScheduleBuildsTask(scheduler),
 143     ["helenos-get-profiles"])
 144
 145
 146 #
 147 # Coastline: get list of harbours (i.e. ported software) and build all of them
 148 # for all HelenOS builds
 149 scheduler.submit("Determining available harbours",
 150     "coastline-get-harbours",
 151      CoastlineGetHarboursTask(config.harbours.split(',')),
 152      ["coastline-checkout"])
 153
 154 scheduler.submit("Schedule harbour tarballs fetches",
 155     "coastline-fetch",
 156     CoastlineScheduleFetchesTask(scheduler),
 157     ["coastline-get-harbours" ])
 158
 159
 160 scheduler.submit("Schedule Coastline builds",
 161     "coastline-build",
 162     CoastlineScheduleBuildsTask(scheduler),
 163     [
 164         # Data dependencies
 165         "helenos-get-profiles", "coastline-get-harbours",
 166         # Task dependencies
 167         "helenos-build", "coastline-fetch"
 168     ])
 169
 170
 171 extra_builds = HelenOSExtraBuildsManager(scheduler)
 172
 173 #
 174 # Tests
 175 scheduler.submit("Determine available test scenarios",
 176     "tests-get-list",
 177     GetTestListTask(config.self_path, config.tests.split(',')),
 178     [])
 179
 180 scheduler.submit("Schedule tests",
 181     "tests-schedule",
 182     ScheduleTestsTask(scheduler, extra_builds, config.self_path),
 183     [
 184         "tests-get-list",
 185         "helenos-build",
 186         "coastline-build"
 187     ])
 188
 189
 190 #
 191 # Wait for all builds (and everything) to complete before creating
 192 # the web page with results.
 193 scheduler.barrier()
 194
 195 scheduler.close_report()
 196
 197 scheduler.submit("Generate HTML report",
 198     "html-report",
 199     MakeHtmlReportTask(config.self_path, config.rss_url, config.web_resource_path))
 200
 201
 202 scheduler.done()