2 # Populate builds using prebuilt packages where possible to speed up builds
3 # and allow staging to be reconstructed.
5 # To use it add that line to conf/local.conf:
7 # INHERIT += "packaged-staging"
12 # bitbake.conf set PSTAGING_ACTIVE = "0", this class sets to "1" if we're active
14 PSTAGE_PKGVERSION = "${PV}-${PR}"
15 PSTAGE_PKGARCH = "${BUILD_SYS}"
16 PSTAGE_EXTRAPATH ?= "/${OELAYOUT_ABI}/${DISTRO_PR}/"
17 PSTAGE_PKGPATH = "${DISTRO}${PSTAGE_EXTRAPATH}"
18 PSTAGE_PKGPN = "${@bb.data.expand('staging-${PN}-${MULTIMACH_ARCH}${TARGET_VENDOR}-${TARGET_OS}', d).replace('_', '-')}"
19 PSTAGE_PKGNAME = "${PSTAGE_PKGPN}_${PSTAGE_PKGVERSION}_${PSTAGE_PKGARCH}.ipk"
20 PSTAGE_PKG = "${PSTAGE_DIR}/${PSTAGE_PKGPATH}/${PSTAGE_PKGNAME}"
21 PSTAGE_WORKDIR = "${TMPDIR}/pstage"
22 PSTAGE_SCAN_CMD ?= "find ${PSTAGE_TMPDIR_STAGE} \( -name "*.la" -o -name "*-config"\) -type f"
24 PSTAGE_NATIVEDEPENDS = "\
29 BB_STAMP_WHITELIST = "${PSTAGE_NATIVEDEPENDS}"
31 def _package_unlink (f):
37 if e.errno == errno.ENOENT:
44 # These classes encode staging paths into the binary data so can only be
45 # reused if the path doesn't change/
46 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('sdk', d) or bb.data.inherits_class('crosssdk', d):
47 path = bb.data.getVar('PSTAGE_PKGPATH', d, 1)
48 path = path + bb.data.getVar('TMPDIR', d, 1).replace('/', '-')
49 bb.data.setVar('PSTAGE_PKGPATH', path, d)
50 scan_cmd = "grep -Irl ${STAGING_DIR} ${PSTAGE_TMDPDIR_STAGE}"
51 bb.data.setVar('PSTAGE_SCAN_CMD', scan_cmd, d)
53 # PSTAGE_NATIVEDEPENDS lists the packages we need before we can use packaged
54 # staging. There will always be some packages we depend on.
55 if bb.data.inherits_class('native', d):
56 pn = bb.data.getVar('PN', d, True)
57 nativedeps = bb.data.getVar('PSTAGE_NATIVEDEPENDS', d, True).split()
59 pstage_allowed = False
61 # Images aren't of interest to us
62 if bb.data.inherits_class('image', d):
63 pstage_allowed = False
65 if bb.data.getVar('PSTAGING_DISABLED', d, True) == "1":
66 pstage_allowed = False
68 # Add task dependencies if we're active, otherwise mark packaged staging
71 deps = bb.data.getVarFlag('do_setscene', 'depends', d) or ""
72 deps += " stagemanager-native:do_populate_sysroot"
73 bb.data.setVarFlag('do_setscene', 'depends', deps, d)
75 policy = bb.data.getVar("BB_STAMP_POLICY", d, True)
76 if policy == "whitelist" or policy == "full":
77 deps = bb.data.getVarFlag('do_setscene', 'recrdeptask', d) or ""
78 deps += " do_setscene"
79 bb.data.setVarFlag('do_setscene', 'recrdeptask', deps, d)
81 bb.data.setVar("PSTAGING_ACTIVE", "1", d)
83 bb.data.setVar("PSTAGING_ACTIVE", "0", d)
86 PSTAGE_MACHCONFIG = "${PSTAGE_WORKDIR}/opkg.conf"
88 PSTAGE_PKGMANAGER = "stage-manager-ipkg"
90 PSTAGE_BUILD_CMD = "stage-manager-ipkg-build -o 0 -g 0"
91 PSTAGE_INSTALL_CMD = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -force-depends -o ${TMPDIR} install"
92 PSTAGE_UPDATE_CMD = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR} update"
93 PSTAGE_REMOVE_CMD = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -force-depends -o ${TMPDIR} remove"
94 PSTAGE_LIST_CMD = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR} list_installed"
96 PSTAGE_TMPDIR_STAGE = "${WORKDIR}/staging-pkg"
98 def pstage_manualclean(srcname, destvarname, d):
99 src = os.path.join(bb.data.getVar('PSTAGE_TMPDIR_STAGE', d, True), srcname)
100 dest = bb.data.getVar(destvarname, d, True)
102 for walkroot, dirs, files in os.walk(src):
103 bb.debug("rm %s" % walkroot)
105 # Avoid breaking the held lock
106 if (file == "staging.lock"):
108 filepath = os.path.join(walkroot, file).replace(src, dest)
109 _package_unlink(filepath)
111 def pstage_set_pkgmanager(d):
112 path = bb.data.getVar("PATH", d, 1)
113 pkgmanager = bb.which(path, 'opkg-cl')
115 pkgmanager = bb.which(path, 'ipkg-cl')
117 bb.data.setVar("PSTAGE_PKGMANAGER", pkgmanager, d)
120 def pstage_cleanpackage(pkgname, d):
121 path = bb.data.getVar("PATH", d, 1)
122 pstage_set_pkgmanager(d)
123 list_cmd = bb.data.getVar("PSTAGE_LIST_CMD", d, True)
125 bb.debug(2, "Checking if staging package installed")
126 lf = bb.utils.lockfile(bb.data.expand("${SYSROOT_LOCK}", d))
127 ret = os.system("PATH=\"%s\" %s | grep %s" % (path, list_cmd, pkgname))
129 bb.debug(1, "Uninstalling package from staging...")
130 removecmd = bb.data.getVar("PSTAGE_REMOVE_CMD", d, 1)
131 ret = os.system("PATH=\"%s\" %s %s" % (path, removecmd, pkgname))
133 bb.note("Failure removing staging package")
135 bb.debug(1, "Manually removing any installed files from staging...")
136 pstage_manualclean("sysroots", "STAGING_DIR", d)
137 pstage_manualclean("cross", "CROSS_DIR", d)
138 pstage_manualclean("deploy", "DEPLOY_DIR", d)
140 bb.utils.unlockfile(lf)
144 Clear the build and temp directories
147 removepkg = bb.data.expand("${PSTAGE_PKGPN}", d)
148 pstage_cleanpackage(removepkg, d)
150 stagepkg = bb.data.expand("${PSTAGE_PKG}", d)
151 bb.note("Removing staging package %s" % base_path_out(stagepkg, d))
152 # Add a wildcard to the end of stagepkg to also get its md5
153 # if it's a fetched package
154 os.system('rm -rf ' + stagepkg + '*')
158 # Assemble appropriate opkg.conf
159 conffile=${PSTAGE_MACHCONFIG}
160 mkdir -p ${PSTAGE_WORKDIR}/pstaging_lists
161 if [ ! -e $conffile ]; then
162 ipkgarchs="${BUILD_SYS}"
164 for arch in $ipkgarchs; do
165 echo "arch $arch $priority" >> $conffile
166 priority=$(expr $priority + 5)
168 echo "dest root /" >> $conffile
170 if [ ! -e ${TMPDIR}${libdir_native}/opkg/info/ ]; then
171 mkdir -p ${TMPDIR}${libdir_native}/opkg/info/
173 if [ ! -e ${TMPDIR}${libdir_native}/ipkg/ ]; then
174 ln -sf opkg/ ${TMPDIR}${libdir_native}/ipkg || true
178 def staging_fetch(stagepkg, d):
181 # only try and fetch if the user has configured a mirror
182 if bb.data.getVar('PSTAGE_MIRROR', d) != "":
183 # Copy the data object and override DL_DIR and SRC_URI
185 dldir = bb.data.expand("${PSTAGE_DIR}/${PSTAGE_PKGPATH}", pd)
186 mirror = bb.data.expand("${PSTAGE_MIRROR}/${PSTAGE_PKGPATH}/", pd)
187 srcuri = mirror + os.path.basename(stagepkg)
188 bb.data.setVar('DL_DIR', dldir, pd)
189 bb.data.setVar('SRC_URI', srcuri, pd)
191 # Try a fetch from the pstage mirror, if it fails just return and
192 # we will build the package
194 bb.fetch.init([srcuri], pd)
195 bb.fetch.go(pd, [srcuri])
199 PSTAGE_TASKS_COVERED = "fetch unpack munge patch configure qa_configure rig_locales compile sizecheck install deploy package populate_sysroot package_write_deb package_write_ipk package_write package_stage qa_staging"
201 SCENEFUNCS += "packagestage_scenefunc"
203 python packagestage_scenefunc () {
205 if bb.data.getVar("PSTAGING_ACTIVE", d, 1) == "0":
208 bb.build.exec_func("staging_helper", d)
210 removepkg = bb.data.expand("${PSTAGE_PKGPN}", d)
212 pstage_cleanpackage(removepkg, d)
214 stagepkg = bb.data.expand("${PSTAGE_PKG}", d)
215 if not os.path.exists(stagepkg):
216 staging_fetch(stagepkg, d)
218 if os.path.exists(stagepkg):
219 path = bb.data.getVar("PATH", d, 1)
220 pstage_set_pkgmanager(d)
221 file = bb.data.getVar("FILE", d, True)
222 bb.debug(2, "Packaged staging active for %s\n" % file)
225 # Install the staging package somewhere temporarily so we can extract the stamp files
227 bb.mkdirhier(bb.data.expand("${WORKDIR}/tstage/${libdir_native}/opkg/info/ ", d))
228 cmd = bb.data.expand("${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -force-depends -o ${WORKDIR}/tstage install", d)
229 ret = os.system("PATH=\"%s\" %s %s" % (path, cmd, stagepkg))
231 bb.fatal("Couldn't install the staging package to a temp directory")
234 # Grab the staging lock now so that we don't have other threads try and
235 # validate or install based on these stamps being valid. This is a
236 # potential issue for certain BB_STAMP_POLICY values and enough
237 # concurrent threads.
239 lf = bb.utils.lockfile(bb.data.expand("${SYSROOT_LOCK}", d))
242 # Copy the stamp files into the main stamps directoy
244 cmd = bb.data.expand("cp -dpR ${WORKDIR}/tstage/stamps/* ${TMPDIR}/stamps/", d)
247 bb.utils.unlockfile(lf)
248 bb.fatal("Couldn't copy the staging package stamp files")
251 # Iterate over the stamps seeing if they're valid. If we find any that
252 # are invalid or the task wasn't in the taskgraph, assume caution and
255 # FIXME - some tasks are safe to ignore in the task graph. e.g. package_write_*
257 taskscovered = bb.data.getVar("PSTAGE_TASKS_COVERED", d, True).split()
258 stamp = bb.data.getVar("STAMP", d, True)
259 for task in taskscovered:
261 stampfn = "%s.%s" % (stamp, task)
262 bb.debug(1, "Checking %s" % (stampfn))
263 if os.path.exists(stampfn):
264 stageok = bb.runqueue.check_stamp_fn(file, task, d)
265 bb.debug(1, "Result %s" % (stageok))
267 bb.utils.unlockfile(lf)
270 # Remove the stamps and files we added above
271 # FIXME - we should really only remove the stamps we added
272 for fname in glob.glob(stamp + '.*'):
273 _package_unlink(fname)
275 os.system(bb.data.expand("rm -rf ${WORKDIR}/tstage", d))
278 bb.note("Staging package found, using it for %s." % file)
279 installcmd = bb.data.getVar("PSTAGE_INSTALL_CMD", d, 1)
280 lf = bb.utils.lockfile(bb.data.expand("${SYSROOT_LOCK}", d))
281 ret = os.system("PATH=\"%s\" %s %s" % (path, installcmd, stagepkg))
282 bb.utils.unlockfile(lf)
284 bb.note("Failure installing prestage package")
286 bb.build.exec_func("staging_package_libtoolhack", d)
288 bb.build.make_stamp("do_stage_package_populated", d)
290 bb.note("Staging package found but invalid for %s" % file)
293 packagestage_scenefunc[cleandirs] = "${PSTAGE_TMPDIR_STAGE}"
294 packagestage_scenefunc[dirs] = "${STAGING_DIR}"
296 addhandler packagedstage_stampfixing_eventhandler
297 python packagedstage_stampfixing_eventhandler() {
298 if bb.event.getName(e) == "StampUpdate":
299 taskscovered = bb.data.getVar("PSTAGE_TASKS_COVERED", e.data, 1).split()
300 for (fn, task) in e.targets:
303 if task in taskscovered:
304 stamp = "%s.do_stage_package_populated" % e.stampPrefix[fn]
305 if os.path.exists(stamp):
306 # We're targetting a task which was skipped with packaged staging
307 # so we need to remove the autogenerated stamps.
308 for task in taskscovered:
309 dir = "%s.do_%s" % (e.stampPrefix[fn], task)
311 _package_unlink(stamp)
314 populate_sysroot_preamble () {
315 if [ "$PSTAGING_ACTIVE" = "1" ]; then
316 stage-manager -p ${STAGING_DIR} -c ${PSTAGE_WORKDIR}/stamp-cache-staging -u || true
317 stage-manager -p ${CROSS_DIR} -c ${PSTAGE_WORKDIR}/stamp-cache-cross -u || true
321 populate_sysroot_postamble () {
322 if [ "$PSTAGING_ACTIVE" = "1" ]; then
323 # list the packages currently installed in staging
324 # ${PSTAGE_LIST_CMD} | awk '{print $1}' > ${PSTAGE_WORKDIR}/installed-list
326 # exitcode == 5 is ok, it means the files change
328 stage-manager -p ${STAGING_DIR} -c ${PSTAGE_WORKDIR}/stamp-cache-staging -u -d ${PSTAGE_TMPDIR_STAGE}/sysroots
330 if [ "$exitcode" != "5" -a "$exitcode" != "0" ]; then
333 stage-manager -p ${CROSS_DIR} -c ${PSTAGE_WORKDIR}/stamp-cache-cross -u -d ${PSTAGE_TMPDIR_STAGE}/cross/${BASE_PACKAGE_ARCH}
334 if [ "$exitcode" != "5" -a "$exitcode" != "0" ]; then
341 packagedstaging_fastpath () {
342 if [ "$PSTAGING_ACTIVE" = "1" ]; then
343 mkdir -p ${PSTAGE_TMPDIR_STAGE}/sysroots/
344 mkdir -p ${PSTAGE_TMPDIR_STAGE}/cross/${BASE_PACKAGE_ARCH}/
345 cp -fpPR ${SYSROOT_DESTDIR}/${STAGING_DIR}/* ${PSTAGE_TMPDIR_STAGE}/sysroots/ || /bin/true
346 cp -fpPR ${SYSROOT_DESTDIR}/${CROSS_DIR}/* ${PSTAGE_TMPDIR_STAGE}/cross/${BASE_PACKAGE_ARCH}/ || /bin/true
350 do_populate_sysroot[dirs] =+ "${PSTAGE_DIR}"
351 python populate_sysroot_prehook() {
352 bb.build.exec_func("populate_sysroot_preamble", d)
355 python populate_sysroot_posthook() {
356 bb.build.exec_func("populate_sysroot_postamble", d)
360 staging_packager () {
362 mkdir -p ${PSTAGE_TMPDIR_STAGE}/CONTROL
363 mkdir -p ${PSTAGE_DIR}/${PSTAGE_PKGPATH}
365 echo "Package: ${PSTAGE_PKGPN}" > ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
366 echo "Version: ${PSTAGE_PKGVERSION}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
367 echo "Description: ${DESCRIPTION}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
368 echo "Section: ${SECTION}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
369 echo "Priority: Optional" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
370 echo "Maintainer: ${MAINTAINER}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
371 echo "Architecture: ${PSTAGE_PKGARCH}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
373 # Protect against empty SRC_URI
375 if [ "$srcuri" == "" ]; then
376 srcuri="OpenEmbedded"
378 echo "Source: ${SRC_URI}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
380 # Deal with libtool not supporting sysroots
381 # Need to remove hardcoded paths and fix these when we install the
383 # Could someone please add sysroot support to libtool!
384 for i in `${PSTAGE_SCAN_CMD}` ; do \
385 sed -i -e s:${STAGING_DIR}:FIXMESTAGINGDIR:g $i
386 echo $i | sed -e 's:${PSTAGE_TMPDIR_STAGE}/::' >> ${PSTAGE_TMPDIR_STAGE}/sysroots/fixmepath
389 ${PSTAGE_BUILD_CMD} ${PSTAGE_TMPDIR_STAGE} ${PSTAGE_DIR}/${PSTAGE_PKGPATH}
392 staging_package_installer () {
393 #${PSTAGE_INSTALL_CMD} ${PSTAGE_PKG}
395 STATUSFILE=${TMPDIR}${libdir_native}/opkg/status
396 echo "Package: ${PSTAGE_PKGPN}" >> $STATUSFILE
397 echo "Version: ${PSTAGE_PKGVERSION}" >> $STATUSFILE
398 echo "Status: install user installed" >> $STATUSFILE
399 echo "Architecture: ${PSTAGE_PKGARCH}" >> $STATUSFILE
400 echo "" >> $STATUSFILE
402 CTRLFILE=${TMPDIR}${libdir_native}/opkg/info/${PSTAGE_PKGPN}.control
403 echo "Package: ${PSTAGE_PKGPN}" > $CTRLFILE
404 echo "Version: ${PSTAGE_PKGVERSION}" >> $CTRLFILE
405 echo "Architecture: ${PSTAGE_PKGARCH}" >> $CTRLFILE
407 cd ${PSTAGE_TMPDIR_STAGE}
408 find -type f | grep -v ./CONTROL | sed -e 's/^\.//' > ${TMPDIR}${libdir_native}/opkg/info/${PSTAGE_PKGPN}.list
411 python staging_package_libtoolhack () {
412 # Deal with libtool not supporting sysroots and add our new
414 tmpdir = bb.data.getVar('TMPDIR', d, True)
415 staging = bb.data.getVar('STAGING_DIR', d, True)
416 fixmefn = staging + "/fixmepath"
418 fixmefd = open(fixmefn,"r")
419 fixmefiles = fixmefd.readlines()
421 os.system('rm -f ' + fixmefn)
422 for file in fixmefiles:
423 os.system("sed -i -e s:FIXMESTAGINGDIR:%s:g %s" % (staging, tmpdir + '/' + file))
428 python do_package_stage () {
429 if bb.data.getVar("PSTAGING_ACTIVE", d, 1) != "1":
433 # Handle deploy/ packages
435 bb.build.exec_func("read_subpackage_metadata", d)
436 stagepath = bb.data.getVar("PSTAGE_TMPDIR_STAGE", d, 1)
437 tmpdir = bb.data.getVar("TMPDIR", d, True)
438 packages = (bb.data.getVar('PACKAGES', d, 1) or "").split()
439 if len(packages) > 0:
440 if bb.data.inherits_class('package_ipk', d):
441 ipkpath = bb.data.getVar('DEPLOY_DIR_IPK', d, True).replace(tmpdir, stagepath)
442 if bb.data.inherits_class('package_deb', d):
443 debpath = bb.data.getVar('DEPLOY_DIR_DEB', d, True).replace(tmpdir, stagepath)
444 if bb.data.inherits_class('package_rpm', d):
445 rpmpath = bb.data.getVar('DEPLOY_DIR_RPM', d, True).replace(tmpdir, stagepath)
448 pkgname = bb.data.getVar('PKG_%s' % pkg, d, 1)
451 arch = bb.data.getVar('PACKAGE_ARCH_%s' % pkg, d, 1)
453 arch = bb.data.getVar('PACKAGE_ARCH', d, 1)
454 pr = bb.data.getVar('PR_%s' % pkg, d, 1)
456 pr = bb.data.getVar('PR', d, 1)
457 if not packaged(pkg, d):
459 if bb.data.inherits_class('package_ipk', d):
460 srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "${DISTRO_PR}" + "_" + arch + ".ipk", d)
461 srcfile = bb.data.expand("${DEPLOY_DIR_IPK}/" + arch + "/" + srcname, d)
462 if os.path.exists(srcfile):
463 destpath = ipkpath + "/" + arch + "/"
464 bb.mkdirhier(destpath)
466 bb.copyfile(srcfile, destpath + srcname)
468 if bb.data.inherits_class('package_deb', d):
470 srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "${DISTRO_PR}" + "_all.deb", d)
472 srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "${DISTRO_PR}" + "_${DPKG_ARCH}.deb", d)
473 srcfile = bb.data.expand("${DEPLOY_DIR_DEB}/" + arch + "/" + srcname, d)
474 if os.path.exists(srcfile):
475 destpath = debpath + "/" + arch + "/"
476 bb.mkdirhier(destpath)
477 bb.copyfile(srcfile, destpath + srcname)
479 if bb.data.inherits_class('package_rpm', d):
480 version = bb.data.getVar('PV', d, 1)
481 version = version.replace('-', '+')
482 bb.data.setVar('RPMPV', version, d)
483 srcname = bb.data.expand(pkgname + "-${RPMPV}-" + pr + "${DISTRO_PR}" + ".${TARGET_ARCH}.rpm", d)
484 srcfile = bb.data.expand("${DEPLOY_DIR_RPM}/" + arch + "/" + srcname, d)
485 if os.path.exists(srcfile):
486 destpath = rpmpath + "/" + arch + "/"
487 bb.mkdirhier(destpath)
488 bb.copyfile(srcfile, destpath + srcname)
492 # Handle stamps/ files
494 stampfn = bb.data.getVar("STAMP", d, True)
495 destdir = os.path.dirname(stampfn.replace(tmpdir, stagepath))
496 bb.mkdirhier(destdir)
497 # We need to include the package_stage stamp in the staging package so create one
498 bb.build.make_stamp("do_package_stage", d)
499 os.system("cp -dpR %s.do_* %s/" % (stampfn, destdir))
501 pstage_set_pkgmanager(d)
502 bb.build.exec_func("staging_helper", d)
503 bb.build.exec_func("staging_packager", d)
504 lf = bb.utils.lockfile(bb.data.expand("${SYSROOT_LOCK}", d))
505 bb.build.exec_func("staging_package_installer", d)
506 bb.utils.unlockfile(lf)
510 # Note an assumption here is that do_deploy runs before do_package_write/do_populate_sysroot
512 addtask package_stage after do_package_write do_populate_sysroot before do_build
514 do_package_stage_all () {
517 do_package_stage_all[recrdeptask] = "do_package_stage"
518 addtask package_stage_all after do_package_stage before do_build