hphp/parser/make-parser.sh

   1 #!/bin/bash
   2
   3 unset CDPATH
   4 DIR="$( cd "$( dirname "$0" )" && pwd )"
   5
   6 # If we're using buck, then we'll be in a sandboxed source directory instead of
   7 # the repo.  We want the path to the repo so we can check in the generated
   8 # parser artifacts.
   9 if [ -n "${FBCODE_DIR}" ]; then
  10   DIR="${FBCODE_DIR}/hphp/parser"
  11 fi
  12
  13 INFILE=hphp.y
  14
  15 if [ -z "${INSTALL_DIR}" ]; then
  16   INSTALL_DIR="${DIR}"
  17   BISON=$(which bison)
  18 else
  19   BISON=$(readlink -f $(ls -t "${FBCODE_DIR}"/third-party2/bison/2.4.1/centos5.2-native/*/bin/bison | head -1))
  20   BISON_DIR=$(dirname $(dirname "${BISON}"))
  21   export BISON_PKGDATADIR="${BISON_DIR}/share/bison"
  22 fi
  23
  24 if [ ! -x "${BISON}" ]; then
  25   echo "bison not found" 1>&2
  26   exit 1
  27 fi
  28
  29 OUTFILE5=${INSTALL_DIR}/hphp.5.tab.cpp
  30 OUTFILE7=${INSTALL_DIR}/hphp.7.tab.cpp
  31 OUTHEADER5=${INSTALL_DIR}/hphp.5.tab.hpp
  32 OUTHEADER7=${INSTALL_DIR}/hphp.7.tab.hpp
  33 OUTHEADER=${INSTALL_DIR}/hphp.tab.hpp
  34
  35 BISON_OPTS="--verbose --locations -d"
  36
  37 TMP=$(mktemp)
  38
  39 # Use a potentially-scary awk script to split the single hphp.y parser into two
  40 # outputs.  It's actually not that bad.
  41 #  - The idea is that anything between a /* !PHP5_ONLY */ line and a /* !END */
  42 #    line are removed from the PHP7 parser, and vice versa for PHP5.
  43 #  - This is done just by looking for those tokens and setting the "flag".
  44 #  - We make sure to still print things for the removed lines, to keep line
  45 #    numbers consistent to ease debugging.
  46
  47 awk \
  48   '/!END/{flag=0} flag{print "/* REMOVED */"} /!PHP7_ONLY/{print; flag=1} !flag' \
  49   "${INFILE}" > "${TMP}"
  50 # shellcheck disable=SC2086
  51 "${BISON}" ${BISON_OPTS} -pCompiler5 "-o${OUTFILE5}" "${TMP}"
  52 if [ $? -ne 0 ]; then
  53   echo "Bison failed to compile PHP5 parser!"
  54   exit 1
  55 fi
  56
  57 awk \
  58   '/!END/{flag=0} flag{print "/* REMOVED */"} /!PHP5_ONLY/{print; flag=1} !flag' \
  59   "${INFILE}" > "${TMP}"
  60 # shellcheck disable=SC2086
  61 "${BISON}" ${BISON_OPTS} -pCompiler7 "-o${OUTFILE7}" "${TMP}"
  62 if [ $? -ne 0 ] ; then
  63   echo "Bison failed to compile PHP7 parser!"
  64   exit 1
  65 fi
  66
  67 rm "${TMP}"
  68
  69 # Remove alpha variance in "#line" directives.
  70 if [ "${INSTALL_DIR}" = "${DIR}" ]; then
  71   sed -i -e "s#${TMP}#${INFILE}#g" -e "s#${DIR}/##g" "${OUTFILE5}"
  72   sed -i -e "s#${TMP}#${INFILE}#g" -e "s#${DIR}/##g" "${OUTFILE7}"
  73 else
  74   sed -i "s#${TMP}#${INFILE}#g" "${OUTFILE5}"
  75   sed -i "s#${TMP}#${INFILE}#g" "${OUTFILE7}"
  76 fi
  77
  78 sed -i \
  79     -e 's@int Compiler[57]parse.*@@' \
  80     -e 's@.*int Compiler[57]debug.*@@' \
  81     -e "s@#ifndef YY_COMPILER[57]_.*@@g" \
  82     -e "s@# define YY_COMPILER[57]_.*@@g" \
  83     -e "s@#endif /\* !YY_COMPILER[57]_.*@@g" \
  84     "${OUTHEADER5}" "${OUTHEADER7}"
  85
  86 cmp "${OUTHEADER5}" "${OUTHEADER7}"
  87 if [ $? -ne 0 ]; then
  88   echo "PHP5 and PHP7 headers differ, must be the same tokens"
  89   exit 1
  90 fi
  91 cp "${OUTHEADER5}" "${OUTHEADER}"
  92
  93 # Lots of our logic relies on knowing the shape of the token table.  Sadly it is
  94 # an enum without introspection, so instead make it macros so we can control its
  95 # shape on re-requires of the .hpp file.
  96 sed -i -r \
  97     -e 's/(T_\w+)\s+=\s+([0-9]+)\s*,?/YYTOKEN(\2, \1)/g' \
  98     -e "s/\s+enum\s+yytokentype/#ifndef YYTOKEN_MAP\n#define YYTOKEN_MAP enum yytokentype\n#define YYTOKEN(num, name) name = num,\n#endif\n   YYTOKEN_MAP/" \
  99     "${OUTHEADER}"
 100
 101 # Remove the include guard's #endif (-e doesn't work for this).
 102 sed -i '$ d' "${OUTHEADER}"
 103
 104 # We don't want to rely on the grammar to have a fixed start and end token, so
 105 # let's parse the file and make two macros for the min and max.
 106 TOKEN_MIN=$(grep "^\s\+YYTOKEN(" "${OUTHEADER}" -m 1 | \
 107   sed -r -e 's/\s+YYTOKEN.([0-9]+).*/#ifndef YYTOKEN_MIN\n#define YYTOKEN_MIN \1\n#endif/')
 108 TOKEN_MAX=$(grep "^\s\+YYTOKEN(" "${OUTHEADER}" | tail -n 1 | \
 109   sed -r -e 's/\s+YYTOKEN.([0-9]+).*/#ifndef YYTOKEN_MAX\n#define YYTOKEN_MAX \1\n#endif/')
 110
 111 echo -e "${TOKEN_MIN}\n\n${TOKEN_MAX}" >> "${OUTHEADER}"
 112
 113 # Renaming some stuff in the cpp file.
 114 sed -i \
 115     -e "s/first_line/line0/g"   \
 116     -e "s/last_line/line1/g"     \
 117     -e "s/first_column/char0/g"  \
 118     -e "s/last_column/char1/g"   \
 119     -e "s/union/struct/g"        \
 120     -e "s/YYSTACK_ALLOC (YYSTACK_BYTES (yystacksize));/YYSTACK_ALLOC (YYSTACK_BYTES (yystacksize));\n      memset(yyptr, 0, YYSTACK_BYTES (yystacksize));/" \
 121     -e "s/YYSTACK_RELOCATE (yyvs_alloc, yyvs)/YYSTACK_RELOCATE_RESET (yyvs_alloc, yyvs)/" \
 122     -e "s/YYSTACK_FREE (yyss)/YYSTACK_FREE (yyss);\n  YYSTACK_CLEANUP/" \
 123     -e "s/\".*hphp\\.\\(.\\)\\.tab\\.cpp\"/\"hphp.\\1.tab.cpp\"/" \
 124     "${OUTFILE5}" "${OUTFILE7}"
 125
 126 # We still want the files in our tree since they are checked in.
 127 if [ "${INSTALL_DIR}" != "${DIR}" ]; then
 128   sed -i -e "1i// @""generated" "${OUTFILE5}"
 129   sed -i -e "1i// @""generated" "${OUTFILE7}"
 130   sed -i -e "1i// @""generated" "${OUTHEADER}"
 131   cp "${OUTFILE5}" "${DIR}/hphp.5.tab.cpp"
 132   cp "${OUTFILE7}" "${DIR}/hphp.7.tab.cpp"
 133   cp "${OUTHEADER}" "${DIR}/hphp.tab.hpp"
 134 fi