lib/fuzzing/oss-fuzz/do_build.sh

   1 #!/bin/sh
   2 #
   3 # This is not a general-purpose build script, but instead one specific
   4 # to the Google oss-fuzz compile environment.
   5 #
   6 # https://google.github.io/oss-fuzz/getting-started/new-project-guide/#Requirements
   7 #
   8 # https://github.com/google/oss-fuzz/blob/master/infra/base-images/base-builder/README.md#provided-environment-variables
   9 #
  10 # This file is run by build_samba.sh, which is run by
  11 # https://github.com/google/oss-fuzz/blob/master/projects/samba/build.sh
  12 # which does nothing else.
  13 #
  14 # We have to push to oss-fuzz CFLAGS into the waf ADDITIONAL_CFLAGS
  15 # as otherwise waf's configure fails linking the first test binary
  16 #
  17 # CFLAGS are supplied by the caller, eg the oss-fuzz compile command
  18 #
  19 # Additional arguments are passed to configure, to allow this to be
  20 # tested in autobuild.py
  21 #
  22
  23 # Ensure we give good trace info, fail right away and fail with unset
  24 # variables
  25 set -e
  26 set -x
  27 set -u
  28
  29 # It is critical that this script, just as the rest of Samba's GitLab
  30 # CI docker has LANG set to en_US.utf8 (oss-fuzz fails to set this)
  31 if [ -f /etc/default/locale ]; then
  32         . /etc/default/locale
  33 elif [ -f /etc/locale.conf ]; then
  34         . /etc/locale.conf
  35 fi
  36 export LANG
  37 export LC_ALL
  38
  39 ADDITIONAL_CFLAGS="$CFLAGS"
  40 export ADDITIONAL_CFLAGS
  41 CFLAGS=""
  42 export CFLAGS
  43 LD="$CXX"
  44 export LD
  45
  46 # Use the system Python, not the OSS-Fuzz provided statically linked
  47 # and instrumented Python, because we can't statically link.
  48
  49 PYTHON=/usr/bin/python3
  50 export PYTHON
  51
  52 # $SANITIZER is provided by the oss-fuzz "compile" command
  53 #
  54 # We need to add the waf configure option as otherwise when we also
  55 # get (eg) -fsanitize=address via the CFLAGS we will fail to link
  56 # correctly
  57
  58 case "$SANITIZER" in
  59 address)
  60         SANITIZER_ARG='--address-sanitizer'
  61         ;;
  62 undefined)
  63         SANITIZER_ARG='--undefined-sanitizer'
  64         ;;
  65 coverage)
  66         # Thankfully clang operating as ld has no objection to the
  67         # cc style options, so we can just set ADDITIONAL_LDFLAGS
  68         # to ensure the coverage build is done, despite waf splitting
  69         # the compile and link phases.
  70         ADDITIONAL_LDFLAGS="${ADDITIONAL_LDFLAGS:-} $COVERAGE_FLAGS"
  71         export ADDITIONAL_LDFLAGS
  72
  73         SANITIZER_ARG=''
  74         ;;
  75 esac
  76
  77 # $LIB_FUZZING_ENGINE is provided by the oss-fuzz "compile" command
  78 #
  79
  80 # --disable-new-dtags linker flag creates fuzzer binaries with RPATH
  81 # header instead of RUNPATH header. Modern linkers use RUNPATH by
  82 # default.
  83 ./configure -C --without-gettext --enable-debug --enable-developer \
  84         --enable-libfuzzer \
  85         $SANITIZER_ARG \
  86         --disable-warnings-as-errors \
  87         --abi-check-disable \
  88         "--fuzz-target-ldflags=-Wl,--disable-new-dtags $LIB_FUZZING_ENGINE" \
  89         --nonshared-binary=ALL \
  90         "$@" \
  91         LINK_CC="$CXX"
  92
  93 make -j
  94
  95 # Make a directory for the system shared libraries to be copied into
  96 mkdir -p $OUT/lib
  97
  98 # oss-fuzz would prefer for all the binaries put into $OUT to be
  99 # statically linked.
 100 #
 101 # We can't static link to all the system libs with waf, so copy the
 102 # libraries we need to $OUT/lib and set the rpath to point there.
 103 # This is similar to how firefox handles this.
 104 #
 105 # NOTE on RPATH vs RUNPATH:
 106 #
 107 # RUNPATH appears to be the more modern version, and so modern ld.bfd
 108 # and ld.gold only set RUNPATH.  RUNPATH makes sense on most systems,
 109 # but not for our hack.
 110 #
 111 # If we use RUNPATH, we can get an error like this:
 112 # Step #6: Error occurred while running fuzz_nmblib_parse_packet:
 113 # Step #6: /workspace/out/coverage/fuzz_nmblib_parse_packet: error while loading shared libraries: libavahi-common.so.3: cannot open shared object file: No such file or directory
 114 #
 115 # This is because the full contents of $OUT are copied to yet another
 116 # host, which otherwise does not have much of linux at all.  oss-fuzz
 117 # prefers a static binary because that will 'just work', but we can't
 118 # do that, so we need to use linker tricks.
 119 #
 120 # If the linker used RUNPATH (eg ld.bfd on Ubuntu 18.04 and later, ld.gold):
 121 # * bin=fuzz_nmblib_parse_packet
 122 # * OUT=/tmp/3/b12207/prefix/samba-fuzz
 123 # * chrpath -r '$ORIGIN/lib' $OUT/$bin'
 124 # * ldd $OUT/$bin
 125 #       linux-vdso.so.1 (0x00007ffd4b7a5000)
 126 #       libasan.so.5 => /tmp/3/b12207/prefix/samba-fuzz/lib/libasan.so.5 (0x00007ff25bdd0000)
 127 #       libldap_r-2.4.so.2 => /tmp/3/b12207/prefix/samba-fuzz/lib/libldap_r-2.4.so.2 (0x00007ff25bd7a000)
 128 #       liblber-2.4.so.2 => /tmp/3/b12207/prefix/samba-fuzz/lib/liblber-2.4.so.2 (0x00007ff25bd69000)
 129 #       libunwind-x86_64.so.8 => /tmp/3/b12207/prefix/samba-fuzz/lib/libunwind-x86_64.so.8 (0x00007ff25bd47000)
 130 #       libunwind.so.8 => /tmp/3/b12207/prefix/samba-fuzz/lib/libunwind.so.8 (0x00007ff25bd2a000)
 131 #       libgnutls.so.30 => /tmp/3/b12207/prefix/samba-fuzz/lib/libgnutls.so.30 (0x00007ff25bb54000)
 132 #       libdl.so.2 => /tmp/3/b12207/prefix/samba-fuzz/lib/libdl.so.2 (0x00007ff25bb4c000)
 133 #       libz.so.1 => /tmp/3/b12207/prefix/samba-fuzz/lib/libz.so.1 (0x00007ff25bb30000)
 134 #       libjansson.so.4 => /tmp/3/b12207/prefix/samba-fuzz/lib/libjansson.so.4 (0x00007ff25bb21000)
 135 #       libresolv.so.2 => /tmp/3/b12207/prefix/samba-fuzz/lib/libresolv.so.2 (0x00007ff25bb05000)
 136 #       libsystemd.so.0 => /tmp/3/b12207/prefix/samba-fuzz/lib/libsystemd.so.0 (0x00007ff25ba58000)
 137 #       libpthread.so.0 => /tmp/3/b12207/prefix/samba-fuzz/lib/libpthread.so.0 (0x00007ff25ba35000)
 138 #       libicuuc.so.66 => /tmp/3/b12207/prefix/samba-fuzz/lib/libicuuc.so.66 (0x00007ff25b84d000)
 139 #       libicui18n.so.66 => /tmp/3/b12207/prefix/samba-fuzz/lib/libicui18n.so.66 (0x00007ff25b54e000)
 140 #       libcap.so.2 => /tmp/3/b12207/prefix/samba-fuzz/lib/libcap.so.2 (0x00007ff25b545000)
 141 #       libbsd.so.0 => /tmp/3/b12207/prefix/samba-fuzz/lib/libbsd.so.0 (0x00007ff25b52b000)
 142 #       libnsl.so.1 => /tmp/3/b12207/prefix/samba-fuzz/lib/libnsl.so.1 (0x00007ff25b50e000)
 143 #       libc.so.6 => /tmp/3/b12207/prefix/samba-fuzz/lib/libc.so.6 (0x00007ff25b31c000)
 144 #       librt.so.1 => /lib/x86_64-linux-gnu/librt.so.1 (0x00007ff25b2f2000)
 145 #       libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007ff25b1a3000)
 146 #       libgcc_s.so.1 => /lib/x86_64-linux-gnu/libgcc_s.so.1 (0x00007ff25b188000)
 147 #       libsasl2.so.2 => /usr/lib/x86_64-linux-gnu/libsasl2.so.2 (0x00007ff25b16b000)
 148 #       libgssapi.so.3 => /usr/lib/x86_64-linux-gnu/libgssapi.so.3 (0x00007ff25b126000)
 149 #       liblzma.so.5 => /lib/x86_64-linux-gnu/liblzma.so.5 (0x00007ff25b0fd000)
 150 #       /lib64/ld-linux-x86-64.so.2 (0x00007ff25ea0c000)
 151 #       libp11-kit.so.0 => /usr/lib/x86_64-linux-gnu/libp11-kit.so.0 (0x00007ff25afc5000)
 152 #       libidn2.so.0 => /usr/lib/x86_64-linux-gnu/libidn2.so.0 (0x00007ff25afa4000)
 153 #       libunistring.so.2 => /usr/lib/x86_64-linux-gnu/libunistring.so.2 (0x00007ff25ae22000)
 154 #       libtasn1.so.6 => /usr/lib/x86_64-linux-gnu/libtasn1.so.6 (0x00007ff25ae0c000)
 155 #       libnettle.so.7 => /usr/lib/x86_64-linux-gnu/libnettle.so.7 (0x00007ff25add2000)
 156 #       libhogweed.so.5 => /usr/lib/x86_64-linux-gnu/libhogweed.so.5 (0x00007ff25ad9a000)
 157 #       libgmp.so.10 => /usr/lib/x86_64-linux-gnu/libgmp.so.10 (0x00007ff25ad14000)
 158 #       liblz4.so.1 => /usr/lib/x86_64-linux-gnu/liblz4.so.1 (0x00007ff25acf3000)
 159 #       libgcrypt.so.20 => /usr/lib/x86_64-linux-gnu/libgcrypt.so.20 (0x00007ff25abd5000)
 160 #       libicudata.so.66 => /usr/lib/x86_64-linux-gnu/libicudata.so.66 (0x00007ff259114000)
 161 #       libstdc++.so.6 => /usr/lib/x86_64-linux-gnu/libstdc++.so.6 (0x00007ff258f33000)
 162 #       libheimntlm.so.0 => /usr/lib/x86_64-linux-gnu/libheimntlm.so.0 (0x00007ff258f25000)
 163 #       libkrb5.so.26 => /usr/lib/x86_64-linux-gnu/libkrb5.so.26 (0x00007ff258e92000)
 164 #       libasn1.so.8 => /usr/lib/x86_64-linux-gnu/libasn1.so.8 (0x00007ff258deb000)
 165 #       libcom_err.so.2 => /lib/x86_64-linux-gnu/libcom_err.so.2 (0x00007ff258de4000)
 166 #       libhcrypto.so.4 => /usr/lib/x86_64-linux-gnu/libhcrypto.so.4 (0x00007ff258dac000)
 167 #       libroken.so.18 => /usr/lib/x86_64-linux-gnu/libroken.so.18 (0x00007ff258d93000)
 168 #       libffi.so.7 => /usr/lib/x86_64-linux-gnu/libffi.so.7 (0x00007ff258d85000)
 169 #       libgpg-error.so.0 => /lib/x86_64-linux-gnu/libgpg-error.so.0 (0x00007ff258d62000)
 170 #       libwind.so.0 => /usr/lib/x86_64-linux-gnu/libwind.so.0 (0x00007ff258d38000)
 171 #       libheimbase.so.1 => /usr/lib/x86_64-linux-gnu/libheimbase.so.1 (0x00007ff258d26000)
 172 #       libhx509.so.5 => /usr/lib/x86_64-linux-gnu/libhx509.so.5 (0x00007ff258cd8000)
 173 #       libsqlite3.so.0 => /usr/lib/x86_64-linux-gnu/libsqlite3.so.0 (0x00007ff258bad000)
 174 #       libcrypt.so.1 => /lib/x86_64-linux-gnu/libcrypt.so.1 (0x00007ff258b72000)
 175 #
 176 # Note how all the dependencies of libc and gnutls are not forced to
 177 # $OUT/lib (via the magic $ORIGIN variable, meaning the directory of
 178 # the binary).  These will not be found on the target system!
 179 #
 180 # If the linker used RPATH however
 181 # * bin=fuzz_nmblib_parse_packet
 182 # * OUT=/tmp/3/b22/prefix/samba-fuzz
 183 # * chrpath -r '$ORIGIN/lib' $OUT/$bin'
 184 # * ldd $OUT/$bin
 185 #       linux-vdso.so.1 =>  (0x00007ffef85c7000)
 186 #       libasan.so.2 => /tmp/3/b22/prefix/samba-fuzz/lib/libasan.so.2 (0x00007f3668b4f000)
 187 #       libldap_r-2.4.so.2 => /tmp/3/b22/prefix/samba-fuzz/lib/libldap_r-2.4.so.2 (0x00007f36688fe000)
 188 #       liblber-2.4.so.2 => /tmp/3/b22/prefix/samba-fuzz/lib/liblber-2.4.so.2 (0x00007f36686ef000)
 189 #       libunwind-x86_64.so.8 => /tmp/3/b22/prefix/samba-fuzz/lib/libunwind-x86_64.so.8 (0x00007f36684d0000)
 190 #       libunwind.so.8 => /tmp/3/b22/prefix/samba-fuzz/lib/libunwind.so.8 (0x00007f36682b5000)
 191 #       libgnutls.so.30 => /tmp/3/b22/prefix/samba-fuzz/lib/libgnutls.so.30 (0x00007f3667f85000)
 192 #       libdl.so.2 => /tmp/3/b22/prefix/samba-fuzz/lib/libdl.so.2 (0x00007f3667d81000)
 193 #       libz.so.1 => /tmp/3/b22/prefix/samba-fuzz/lib/libz.so.1 (0x00007f3667b67000)
 194 #       libjansson.so.4 => /tmp/3/b22/prefix/samba-fuzz/lib/libjansson.so.4 (0x00007f366795a000)
 195 #       libresolv.so.2 => /tmp/3/b22/prefix/samba-fuzz/lib/libresolv.so.2 (0x00007f366773f000)
 196 #       libsystemd.so.0 => /tmp/3/b22/prefix/samba-fuzz/lib/libsystemd.so.0 (0x00007f366be2f000)
 197 #       libpthread.so.0 => /tmp/3/b22/prefix/samba-fuzz/lib/libpthread.so.0 (0x00007f3667522000)
 198 #       libicuuc.so.55 => /tmp/3/b22/prefix/samba-fuzz/lib/libicuuc.so.55 (0x00007f366718e000)
 199 #       libicui18n.so.55 => /tmp/3/b22/prefix/samba-fuzz/lib/libicui18n.so.55 (0x00007f3666d2c000)
 200 #       libcap.so.2 => /tmp/3/b22/prefix/samba-fuzz/lib/libcap.so.2 (0x00007f3666b26000)
 201 #       libbsd.so.0 => /tmp/3/b22/prefix/samba-fuzz/lib/libbsd.so.0 (0x00007f3666911000)
 202 #       libnsl.so.1 => /tmp/3/b22/prefix/samba-fuzz/lib/libnsl.so.1 (0x00007f36666f8000)
 203 #       libc.so.6 => /tmp/3/b22/prefix/samba-fuzz/lib/libc.so.6 (0x00007f366632e000)
 204 #       libm.so.6 => /tmp/3/b22/prefix/samba-fuzz/lib/libm.so.6 (0x00007f3666025000)
 205 #       libgcc_s.so.1 => /tmp/3/b22/prefix/samba-fuzz/lib/libgcc_s.so.1 (0x00007f3665e0f000)
 206 #       libsasl2.so.2 => /tmp/3/b22/prefix/samba-fuzz/lib/libsasl2.so.2 (0x00007f3665bf4000)
 207 #       libgssapi.so.3 => /tmp/3/b22/prefix/samba-fuzz/lib/libgssapi.so.3 (0x00007f36659b3000)
 208 #       liblzma.so.5 => /tmp/3/b22/prefix/samba-fuzz/lib/liblzma.so.5 (0x00007f3665791000)
 209 #       /lib64/ld-linux-x86-64.so.2 (0x00007f366bc93000)
 210 #       libp11-kit.so.0 => /tmp/3/b22/prefix/samba-fuzz/lib/libp11-kit.so.0 (0x00007f366552d000)
 211 #       libidn.so.11 => /tmp/3/b22/prefix/samba-fuzz/lib/libidn.so.11 (0x00007f36652fa000)
 212 #       libtasn1.so.6 => /tmp/3/b22/prefix/samba-fuzz/lib/libtasn1.so.6 (0x00007f36650e7000)
 213 #       libnettle.so.6 => /tmp/3/b22/prefix/samba-fuzz/lib/libnettle.so.6 (0x00007f3664eb1000)
 214 #       libhogweed.so.4 => /tmp/3/b22/prefix/samba-fuzz/lib/libhogweed.so.4 (0x00007f3664c7e000)
 215 #       libgmp.so.10 => /tmp/3/b22/prefix/samba-fuzz/lib/libgmp.so.10 (0x00007f36649fe000)
 216 #       libselinux.so.1 => /tmp/3/b22/prefix/samba-fuzz/lib/libselinux.so.1 (0x00007f36647dc000)
 217 #       librt.so.1 => /tmp/3/b22/prefix/samba-fuzz/lib/librt.so.1 (0x00007f36645d4000)
 218 #       libgcrypt.so.20 => /tmp/3/b22/prefix/samba-fuzz/lib/libgcrypt.so.20 (0x00007f36642f3000)
 219 #       libicudata.so.55 => /tmp/3/b22/prefix/samba-fuzz/lib/libicudata.so.55 (0x00007f366283c000)
 220 #       libstdc++.so.6 => /tmp/3/b22/prefix/samba-fuzz/lib/libstdc++.so.6 (0x00007f36624ba000)
 221 #       libheimntlm.so.0 => /tmp/3/b22/prefix/samba-fuzz/lib/libheimntlm.so.0 (0x00007f36622b1000)
 222 #       libkrb5.so.26 => /tmp/3/b22/prefix/samba-fuzz/lib/libkrb5.so.26 (0x00007f3662027000)
 223 #       libasn1.so.8 => /tmp/3/b22/prefix/samba-fuzz/lib/libasn1.so.8 (0x00007f3661d85000)
 224 #       libcom_err.so.2 => /tmp/3/b22/prefix/samba-fuzz/lib/libcom_err.so.2 (0x00007f3661b81000)
 225 #       libhcrypto.so.4 => /tmp/3/b22/prefix/samba-fuzz/lib/libhcrypto.so.4 (0x00007f366194e000)
 226 #       libroken.so.18 => /tmp/3/b22/prefix/samba-fuzz/lib/libroken.so.18 (0x00007f3661738000)
 227 #       libffi.so.6 => /tmp/3/b22/prefix/samba-fuzz/lib/libffi.so.6 (0x00007f3661530000)
 228 #       libpcre.so.3 => /tmp/3/b22/prefix/samba-fuzz/lib/libpcre.so.3 (0x00007f36612c0000)
 229 #       libgpg-error.so.0 => /tmp/3/b22/prefix/samba-fuzz/lib/libgpg-error.so.0 (0x00007f36610ac000)
 230 #       libwind.so.0 => /tmp/3/b22/prefix/samba-fuzz/lib/libwind.so.0 (0x00007f3660e83000)
 231 #       libheimbase.so.1 => /tmp/3/b22/prefix/samba-fuzz/lib/libheimbase.so.1 (0x00007f3660c74000)
 232 #       libhx509.so.5 => /tmp/3/b22/prefix/samba-fuzz/lib/libhx509.so.5 (0x00007f3660a29000)
 233 #       libsqlite3.so.0 => /tmp/3/b22/prefix/samba-fuzz/lib/libsqlite3.so.0 (0x00007f3660754000)
 234 #       libcrypt.so.1 => /tmp/3/b22/prefix/samba-fuzz/lib/libcrypt.so.1 (0x00007f366051c000)
 235 #
 236 # See how the runtime linker seems to honour the RPATH for
 237 # dependencies of dependencies in this case.  This helps us us lot.
 238
 239 for x in bin/fuzz_*; do
 240         # Copy any system libraries needed by this fuzzer to $OUT/lib.
 241
 242         # We run ldd on $x, the fuzz_binary in bin/ which has not yet had
 243         # the RPATH altered.  This is clearer for debugging in local
 244         # development builds as $OUT is not cleaned between runs.
 245         #
 246         # Otherwise trying to re-run this can see cp can fail with:
 247         # cp: '/out/lib/libgcc_s.so.1' and '/out/lib/libgcc_s.so.1' are the same file
 248         # which is really confusing!
 249
 250         # The cut for ( and ' ' removes the special case references to:
 251         #       linux-vdso.so.1 =>  (0x00007ffe8f2b2000)
 252         #   /lib64/ld-linux-x86-64.so.2 (0x00007fc63ea6f000)
 253
 254         ldd $x | cut -f 2 -d '>' | cut -f 1 -d \( | cut -f 2 -d ' ' | xargs -i cp \{\} $OUT/lib/
 255
 256         cp $x $OUT/
 257         bin=$(basename $x)
 258
 259         # This means the copied libraries are found on the runner.
 260         #
 261         # The binaries should we built with RPATH, not RUNPATH, to allow
 262         # libraries used by libraries to be found. This command retains the
 263         # RPATH/RUNPATH header and only changes the path. We later verify this
 264         # in the check_build.sh script.
 265         chrpath -r '$ORIGIN/lib' $OUT/$bin
 266
 267         # Truncate the original binary to save space
 268         echo -n >$x
 269
 270 done
 271
 272 # Strip RUNPATH: or RPATH: entries from shared libraries copied over to $OUT/lib.
 273 # When those libraries get loaded and have further dependencies, a RUNPATH: header
 274 # will cause the dynamic linker to search in the runpath, and not in $OUT/lib,
 275 # and there's no way it will be found in the fuzzing env.
 276 #
 277 # So how is the indirect dependency found in $OUT/lib? Well, suppose the fuzzer binary
 278 # links library A which links library B. During linking, both A and B as listed in the
 279 # executable file's runtime dependencies (This was pioneered in Fedora 13 in 2010, but
 280 # is common behavior now). So we have the fuzzer binary with RPATH set to $OUT/lib, and
 281 # a dependency on library B, and it will therefore find library B in $OUT/lib. On the
 282 # hand, if we keep the RUNPATH in library A, and load A first, it will try loading
 283 # library B as a dependency of A from the wrong place.
 284 chrpath -d $OUT/lib/*
 285
 286 # Grab the seeds dictionary from github and put the seed zips in place
 287 # beside their executables.
 288
 289 wget https://gitlab.com/samba-team/samba-fuzz-seeds/-/jobs/artifacts/master/download?job=zips \
 290         -O seeds.zip
 291
 292 # We might not have unzip, but we do have python
 293 $PYTHON -mzipfile -e seeds.zip $OUT
 294 rm -f seeds.zip