taskcluster/scripts/builder/build-haz-linux.sh

   1 #!/bin/bash -ex
   2
   3 function usage() {
   4     echo "Usage: $0 [--project <js|browser>] <workspace-dir> flags..."
   5     echo "flags are treated the same way as a commit message would be"
   6     echo "(as in, they are scanned for directives just like a try: ... line)"
   7 }
   8
   9 PROJECT=js
  10 WORKSPACE=
  11 while [[ $# -gt 0 ]]; do
  12     if [[ "$1" == "-h" ]] || [[ "$1" == "--help" ]]; then
  13         usage
  14         exit 0
  15     elif [[ "$1" == "--project" ]]; then
  16         shift
  17         PROJECT="$1"
  18         shift
  19     elif [[ "$1" == "--no-tooltool" ]]; then
  20         shift
  21     elif [[ -z "$WORKSPACE" ]]; then
  22         WORKSPACE=$( cd "$1" && pwd )
  23         shift
  24         break
  25     fi
  26 done
  27
  28 function check_commit_msg () {
  29     ( set +e;
  30     if [[ -n "$AUTOMATION" ]]; then
  31         hg --cwd "$GECKO_PATH" log -r. --template '{desc}\n' | grep -F -q -- "$1"
  32     else
  33         echo -- "$SCRIPT_FLAGS" | grep -F -q -- "$1"
  34     fi
  35     )
  36 }
  37
  38 if check_commit_msg "--dep"; then
  39     HAZ_DEP=1
  40 fi
  41
  42 SCRIPT_FLAGS=$*
  43
  44 ANALYSIS_DIR="$WORKSPACE/haz-$PROJECT"
  45
  46 # Ensure all the scripts in this dir are on the path....
  47 DIRNAME=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
  48 PATH=$DIRNAME:$PATH
  49
  50 # Use GECKO_BASE_REPOSITORY as a signal for whether we are running in automation.
  51 export AUTOMATION=${GECKO_BASE_REPOSITORY:+1}
  52
  53 : "${GECKO_PATH:="$DIRNAME"/../../..}"
  54
  55 if ! [ -d "$GECKO_PATH" ]; then
  56     echo "GECKO_PATH must be set to a directory containing a gecko source checkout" >&2
  57     exit 1
  58 fi
  59
  60 # Directory to hold the compiled JS shell that will run the analysis.
  61 HAZARD_SHELL_OBJDIR=$WORKSPACE/obj-haz-shell
  62
  63 export NO_MERCURIAL_SETUP_CHECK=1
  64
  65 if [[ "$PROJECT" = "browser" ]]; then (
  66     cd "$WORKSPACE"
  67     set "$WORKSPACE"
  68     # Mozbuild config:
  69     export MOZBUILD_STATE_PATH=$WORKSPACE/mozbuild/
  70     # Create .mozbuild so mach doesn't complain about this
  71     mkdir -p "$MOZBUILD_STATE_PATH"
  72 ) fi
  73
  74 # Build the shell
  75 export HAZARD_SHELL_OBJDIR # This will be picked up by mozconfig.haz_shell.
  76 $GECKO_PATH/mach hazards build-shell
  77
  78 # Run a self-test
  79 $GECKO_PATH/mach hazards self-test --shell-objdir="$HAZARD_SHELL_OBJDIR"
  80
  81 # Artifacts folder is outside of the cache.
  82 mkdir -p "$HOME"/artifacts/ || true
  83
  84 function grab_artifacts () {
  85     local artifacts
  86     artifacts="$HOME/artifacts"
  87
  88     [ -d "$ANALYSIS_DIR" ] && (
  89         cd "$ANALYSIS_DIR"
  90         ls -lah
  91
  92         # Do not error out if no files found
  93         shopt -s nullglob
  94         set +e
  95         local important
  96         important=(refs.txt unnecessary.txt hazards.txt gcFunctions.txt allFunctions.txt heapWriteHazards.txt rootingHazards.json hazards.html)
  97
  98         # Bundle up the less important but still useful intermediate outputs,
  99         # just to cut down on the clutter in treeherder's Job Details pane.
 100         tar -acvf "${artifacts}/hazardIntermediates.tar.xz" --exclude-from <(IFS=$'\n'; echo "${important[*]}") *.txt *.lst build_xgill.log
 101
 102         # Upload the important outputs individually, so that they will be
 103         # visible in Job Details and accessible to automated jobs.
 104         for f in "${important[@]}"; do
 105             gzip -9 -c "$f" > "${artifacts}/$f.gz"
 106         done
 107
 108         # Check whether the user requested .xdb file upload in the top commit comment
 109         if check_commit_msg "--upload-xdbs"; then
 110             HAZ_UPLOAD_XDBS=1
 111         fi
 112
 113         if [ -n "$HAZ_UPLOAD_XDBS" ]; then
 114             for f in *.xdb; do
 115                 xz -c "$f" > "${artifacts}/$f.bz2"
 116             done
 117         fi
 118     )
 119 }
 120
 121 function check_hazards () {
 122     (
 123     set +e
 124     NUM_HAZARDS=$(grep -c 'Function.*has unrooted.*live across GC call' "$1"/hazards.txt)
 125     NUM_UNSAFE=$(grep -c '^Function.*takes unsafe address of unrooted' "$1"/refs.txt)
 126     NUM_UNNECESSARY=$(grep -c '^Function.* has unnecessary root' "$1"/unnecessary.txt)
 127     NUM_DROPPED=$(grep -c '^Dropped CFG' "$1"/build_xgill.log)
 128     NUM_WRITE_HAZARDS=$(perl -lne 'print $1 if m!found (\d+)/\d+ allowed errors!' "$1"/heapWriteHazards.txt)
 129     NUM_MISSING=$(grep -c '^Function.*expected hazard.*but none were found' "$1"/hazards.txt)
 130
 131     set +x
 132     echo "TinderboxPrint: rooting hazards<br/>$NUM_HAZARDS"
 133     echo "TinderboxPrint: (unsafe references to unrooted GC pointers)<br/>$NUM_UNSAFE"
 134     echo "TinderboxPrint: (unnecessary roots)<br/>$NUM_UNNECESSARY"
 135     echo "TinderboxPrint: missing expected hazards<br/>$NUM_MISSING"
 136     echo "TinderboxPrint: heap write hazards<br/>$NUM_WRITE_HAZARDS"
 137
 138     # Display errors in a way that will get picked up by the taskcluster scraper.
 139     perl -lne 'print "TEST-UNEXPECTED-FAIL | hazards | $1 $2" if /^Function.* has (unrooted .*live across GC call).* (at .*)$/' "$1"/hazards.txt
 140
 141     exit_status=0
 142
 143     if [ $NUM_HAZARDS -gt 0 ]; then
 144         echo "TEST-UNEXPECTED-FAIL | hazards | $NUM_HAZARDS rooting hazards detected" >&2
 145         echo "TinderboxPrint: documentation<br/><a href='https://firefox-source-docs.mozilla.org/js/HazardAnalysis/#diagnosing-a-rooting-hazards-failure'>static rooting hazard analysis failures</a>, visit \"Inspect Task\" link for hazard details"
 146         exit_status=1
 147     fi
 148
 149     if [ $NUM_MISSING -gt 0 ]; then
 150         echo "TEST-UNEXPECTED-FAIL | hazards | $NUM_MISSING expected hazards went undetected" >&2
 151         echo "TinderboxPrint: documentation<br/><a href='https://firefox-source-docs.mozilla.org/js/HazardAnalysis/#diagnosing-a-rooting-hazards-failure'>static rooting hazard analysis failures</a>, visit \"Inspect Task\" link for hazard details"
 152         exit_status=1
 153     fi
 154
 155     NUM_ALLOWED_WRITE_HAZARDS=0
 156     if [ $NUM_WRITE_HAZARDS -gt $NUM_ALLOWED_WRITE_HAZARDS ]; then
 157         echo "TEST-UNEXPECTED-FAIL | heap-write-hazards | $NUM_WRITE_HAZARDS heap write hazards detected out of $NUM_ALLOWED_WRITE_HAZARDS allowed" >&2
 158         echo "TinderboxPrint: documentation<br/><a href='https://firefox-source-docs.mozilla.org/js/HazardAnalysis/#diagnosing-a-heap-write-hazard-failure'>heap write hazard analysis failures</a>, visit \"Inspect Task\" link for hazard details"
 159         exit_status = 1
 160     fi
 161
 162     if [ $NUM_DROPPED -gt 0 ]; then
 163         echo "TEST-UNEXPECTED-FAIL | hazards | $NUM_DROPPED CFGs dropped" >&2
 164         echo "TinderboxPrint: sixgill unable to handle constructs<br/>$NUM_DROPPED"
 165         exit_status=1
 166     fi
 167
 168     if [ $exit_status -ne 0 ]; then
 169         exit $exit_status
 170     fi
 171     )
 172 }
 173
 174 trap grab_artifacts EXIT
 175
 176 # Gather the information from the source tree by compiling it.
 177 $GECKO_PATH/mach hazards gather --project=$PROJECT --work-dir="$ANALYSIS_DIR"
 178
 179 # Analyze the collected information.
 180 $GECKO_PATH/mach hazards analyze --project=$PROJECT --shell-objdir="$HAZARD_SHELL_OBJDIR" --work-dir="$ANALYSIS_DIR"
 181
 182 check_hazards "$ANALYSIS_DIR"
 183
 184 ################################### script end ###################################