hphp/compiler/parser/parser.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-2014 Facebook, Inc. (http://www.facebook.com)     |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16 #include "hphp/compiler/parser/parser.h"
  17 #include <vector>
  18
  19 #include "hphp/compiler/type_annotation.h"
  20 #include "hphp/parser/hphp.tab.hpp"
  21 #include "hphp/compiler/analysis/file_scope.h"
  22
  23 #include "hphp/compiler/expression/expression_list.h"
  24 #include "hphp/compiler/expression/assignment_expression.h"
  25 #include "hphp/compiler/expression/simple_variable.h"
  26 #include "hphp/compiler/expression/dynamic_variable.h"
  27 #include "hphp/compiler/expression/static_member_expression.h"
  28 #include "hphp/compiler/expression/array_element_expression.h"
  29 #include "hphp/compiler/expression/dynamic_function_call.h"
  30 #include "hphp/compiler/expression/simple_function_call.h"
  31 #include "hphp/compiler/expression/scalar_expression.h"
  32 #include "hphp/compiler/expression/object_property_expression.h"
  33 #include "hphp/compiler/expression/object_method_expression.h"
  34 #include "hphp/compiler/expression/list_assignment.h"
  35 #include "hphp/compiler/expression/new_object_expression.h"
  36 #include "hphp/compiler/expression/include_expression.h"
  37 #include "hphp/compiler/expression/unary_op_expression.h"
  38 #include "hphp/compiler/expression/binary_op_expression.h"
  39 #include "hphp/compiler/expression/qop_expression.h"
  40 #include "hphp/compiler/expression/array_pair_expression.h"
  41 #include "hphp/compiler/expression/class_constant_expression.h"
  42 #include "hphp/compiler/expression/parameter_expression.h"
  43 #include "hphp/compiler/expression/modifier_expression.h"
  44 #include "hphp/compiler/expression/constant_expression.h"
  45 #include "hphp/compiler/expression/encaps_list_expression.h"
  46 #include "hphp/compiler/expression/closure_expression.h"
  47 #include "hphp/compiler/expression/yield_expression.h"
  48 #include "hphp/compiler/expression/await_expression.h"
  49 #include "hphp/compiler/expression/user_attribute.h"
  50 #include "hphp/compiler/expression/query_expression.h"
  51 #include "hphp/compiler/expression/simple_query_clause.h"
  52 #include "hphp/compiler/expression/join_clause.h"
  53 #include "hphp/compiler/expression/group_clause.h"
  54 #include "hphp/compiler/expression/ordering.h"
  55
  56 #include "hphp/compiler/statement/function_statement.h"
  57 #include "hphp/compiler/statement/class_statement.h"
  58 #include "hphp/compiler/statement/interface_statement.h"
  59 #include "hphp/compiler/statement/class_variable.h"
  60 #include "hphp/compiler/statement/class_constant.h"
  61 #include "hphp/compiler/statement/method_statement.h"
  62 #include "hphp/compiler/statement/statement_list.h"
  63 #include "hphp/compiler/statement/block_statement.h"
  64 #include "hphp/compiler/statement/if_branch_statement.h"
  65 #include "hphp/compiler/statement/if_statement.h"
  66 #include "hphp/compiler/statement/while_statement.h"
  67 #include "hphp/compiler/statement/do_statement.h"
  68 #include "hphp/compiler/statement/for_statement.h"
  69 #include "hphp/compiler/statement/switch_statement.h"
  70 #include "hphp/compiler/statement/case_statement.h"
  71 #include "hphp/compiler/statement/break_statement.h"
  72 #include "hphp/compiler/statement/continue_statement.h"
  73 #include "hphp/compiler/statement/return_statement.h"
  74 #include "hphp/compiler/statement/global_statement.h"
  75 #include "hphp/compiler/statement/static_statement.h"
  76 #include "hphp/compiler/statement/echo_statement.h"
  77 #include "hphp/compiler/statement/unset_statement.h"
  78 #include "hphp/compiler/statement/exp_statement.h"
  79 #include "hphp/compiler/statement/foreach_statement.h"
  80 #include "hphp/compiler/statement/catch_statement.h"
  81 #include "hphp/compiler/statement/try_statement.h"
  82 #include "hphp/compiler/statement/finally_statement.h"
  83 #include "hphp/compiler/statement/throw_statement.h"
  84 #include "hphp/compiler/statement/goto_statement.h"
  85 #include "hphp/compiler/statement/label_statement.h"
  86 #include "hphp/compiler/statement/use_trait_statement.h"
  87 #include "hphp/compiler/statement/trait_require_statement.h"
  88 #include "hphp/compiler/statement/trait_prec_statement.h"
  89 #include "hphp/compiler/statement/trait_alias_statement.h"
  90 #include "hphp/compiler/statement/typedef_statement.h"
  91
  92 #include "hphp/compiler/analysis/function_scope.h"
  93
  94 #include "hphp/compiler/analysis/code_error.h"
  95 #include "hphp/compiler/analysis/analysis_result.h"
  96
  97 #include "hphp/util/lock.h"
  98 #include "hphp/util/logger.h"
  99 #include "hphp/util/text-util.h"
 100 #include "hphp/util/string-vsnprintf.h"
 101
 102 #include "hphp/runtime/base/file-repository.h"
 103
 104 #ifdef FACEBOOK
 105 #include "hphp/facebook/src/compiler/fb_compiler_hooks.h"
 106 #define RealSimpleFunctionCall FBSimpleFunctionCall
 107 #else
 108 #define RealSimpleFunctionCall SimpleFunctionCall
 109 #endif
 110
 111 #define NEW_EXP0(cls)                                           \
 112   cls##Ptr(new cls(BlockScopePtr(),                             \
 113                    getLocation()))
 114 #define NEW_EXP(cls, e...)                                      \
 115   cls##Ptr(new cls(BlockScopePtr(),                             \
 116                    getLocation(), ##e))
 117 #define NEW_STMT0(cls)                                          \
 118   cls##Ptr(new cls(BlockScopePtr(), getLabelScope(),            \
 119                    getLocation()))
 120 #define NEW_STMT(cls, e...)                                     \
 121   cls##Ptr(new cls(BlockScopePtr(), getLabelScope(),            \
 122                    getLocation(), ##e))
 123
 124 #define PARSE_ERROR(fmt, args...)  HPHP_PARSER_ERROR(fmt, this, ##args)
 125
 126 using namespace HPHP::Compiler;
 127
 128 namespace HPHP {
 129
 130 SimpleFunctionCallPtr NewSimpleFunctionCall(
 131   EXPRESSION_CONSTRUCTOR_PARAMETERS,
 132   const std::string &name, bool hadBackslash, ExpressionListPtr params,
 133   ExpressionPtr cls) {
 134   return SimpleFunctionCallPtr(
 135     new RealSimpleFunctionCall(
 136       EXPRESSION_CONSTRUCTOR_DERIVED_PARAMETER_VALUES,
 137       name, hadBackslash, params, cls));
 138 }
 139
 140 namespace Compiler {
 141 ///////////////////////////////////////////////////////////////////////////////
 142 // statics
 143
 144 StatementListPtr Parser::ParseString(const String& input, AnalysisResultPtr ar,
 145                                      const char *fileName /* = NULL */,
 146                                      bool lambdaMode /* = false */) {
 147   assert(!input.empty());
 148   if (!fileName || !*fileName) fileName = "string";
 149
 150   int len = input.size();
 151   Scanner scanner(input.data(), len, Option::GetScannerType(), fileName, true);
 152   Parser parser(scanner, fileName, ar, len);
 153   parser.m_lambdaMode = lambdaMode;
 154   if (parser.parse()) {
 155     return parser.m_file->getStmt();
 156   }
 157   Logger::Error("Error parsing %s: %s\n%s\n", fileName,
 158                 parser.getMessage().c_str(), input.data());
 159   return StatementListPtr();
 160 }
 161
 162 ///////////////////////////////////////////////////////////////////////////////
 163
 164 Parser::Parser(Scanner &scanner, const char *fileName,
 165                AnalysisResultPtr ar, int fileSize /* = 0 */)
 166     : ParserBase(scanner, fileName), m_ar(ar), m_lambdaMode(false),
 167       m_closureGenerator(false), m_nsState(SeenNothing),
 168       m_aliasTable(getAutoAliasedClasses(), [&] { return isAutoAliasOn(); }) {
 169   string md5str = Eval::FileRepository::unitMd5(scanner.getMd5());
 170   MD5 md5 = MD5(md5str.c_str());
 171
 172   m_file = FileScopePtr(new FileScope(m_fileName, fileSize, md5));
 173
 174   newScope();
 175   m_staticVars.push_back(StringToExpressionPtrVecMap());
 176   m_inTrait = false;
 177
 178   Lock lock(m_ar->getMutex());
 179   m_ar->addFileScope(m_file);
 180 }
 181
 182 bool Parser::parse() {
 183   try {
 184     if (!parseImpl()) {
 185       throw ParseTimeFatalException(m_fileName, line1(),
 186                                     "Parse error: %s",
 187                                     errString().c_str());
 188     }
 189     return true;
 190   } catch (const ParseTimeFatalException& e) {
 191     m_file->cleanupForError(m_ar);
 192     if (e.m_parseFatal) {
 193       m_file->makeParseFatal(m_ar, e.getMessage(), e.m_line);
 194     } else {
 195       m_file->makeFatal(m_ar, e.getMessage(), e.m_line);
 196     }
 197     return false;
 198   }
 199 }
 200
 201 void Parser::error(const char* fmt, ...) {
 202   va_list ap;
 203   va_start(ap, fmt);
 204   string msg;
 205   string_vsnprintf(msg, fmt, ap);
 206   va_end(ap);
 207
 208   fatal(&m_loc, msg.c_str());
 209 }
 210
 211 void Parser::parseFatal(const Location* loc, const char* msg) {
 212   // we can't use loc->file, as the bison parser doesn't track that in YYLTYPE
 213   auto file = m_file->getName().c_str();
 214   auto exn = ParseTimeFatalException(file, loc->line0, "%s", msg);
 215   exn.setParseFatal();
 216   throw exn;
 217 }
 218
 219 void Parser::fatal(const Location* loc, const char* msg) {
 220   throw ParseTimeFatalException(loc->file, loc->line0, "%s", msg);
 221 }
 222
 223 string Parser::errString() {
 224   return m_error.empty() ? getMessage() : m_error;
 225 }
 226
 227 void Parser::pushComment() {
 228   m_comments.push_back(m_scanner.detachDocComment());
 229 }
 230
 231 void Parser::pushComment(const std::string& s) {
 232   m_comments.push_back(s);
 233 }
 234
 235 std::string Parser::popComment() {
 236   std::string ret = m_comments.back();
 237   m_comments.pop_back();
 238   return ret;
 239 }
 240
 241 void Parser::newScope() {
 242   m_scopes.push_back(BlockScopePtrVec());
 243 }
 244
 245 void Parser::completeScope(BlockScopePtr inner) {
 246   always_assert(inner);
 247   BlockScopePtrVec &sv = m_scopes.back();
 248   for (int i = 0, n = sv.size(); i < n; i++) {
 249     BlockScopePtr scope = sv[i];
 250     scope->setOuterScope(inner);
 251   }
 252   inner->getStmt()->resetScope(inner);
 253   m_scopes.pop_back();
 254   if (m_scopes.size()) {
 255     m_scopes.back().push_back(inner);
 256   }
 257 }
 258
 259 LabelScopePtr Parser::getLabelScope() const {
 260   assert(!m_labelScopes.empty());
 261   assert(!m_labelScopes.back().empty());
 262   assert(m_labelScopes.back().back() != nullptr);
 263   return m_labelScopes.back().back();
 264 }
 265
 266 void Parser::onNewLabelScope(bool fresh) {
 267   if (fresh) {
 268     m_labelScopes.push_back(LabelScopePtrVec());
 269   }
 270   assert(!m_labelScopes.empty());
 271   LabelScopePtr labelScope(new LabelScope());
 272   m_labelScopes.back().push_back(labelScope);
 273 }
 274
 275 void Parser::onScopeLabel(const Token& stmt, const Token& label) {
 276   assert(!m_labelScopes.empty());
 277   assert(!m_labelScopes.back().empty());
 278   for (auto& scope : m_labelScopes.back()) {
 279     scope->addLabel(stmt.stmt, label.text());
 280   }
 281 }
 282
 283 void Parser::onCompleteLabelScope(bool fresh) {
 284   assert(!m_labelScopes.empty());
 285   assert(!m_labelScopes.back().empty());
 286   m_labelScopes.back().pop_back();
 287   if (fresh) {
 288     m_labelScopes.pop_back();
 289   }
 290 }
 291
 292 ///////////////////////////////////////////////////////////////////////////////
 293 // variables
 294
 295 void Parser::onName(Token &out, Token &name, NameKind kind) {
 296   switch (kind) {
 297     case StringName:
 298     case StaticName:
 299       onScalar(out, T_STRING, name);
 300       break;
 301     case StaticClassExprName:
 302     case ExprName:
 303     case VarName:
 304       out = name;
 305       break;
 306   }
 307 }
 308
 309 void Parser::onStaticVariable(Token &out, Token *exprs, Token &var,
 310                               Token *value) {
 311   onVariable(out, exprs, var, value);
 312   if (m_staticVars.size()) {
 313     StringToExpressionPtrVecMap &m = m_staticVars.back();
 314     m[var->text()].push_back(out->exp);
 315   }
 316 }
 317
 318 void Parser::onClassVariable(Token &out, Token *exprs, Token &var,
 319                              Token *value) {
 320   onVariable(out, exprs, var, value, false, m_scanner.detachDocComment());
 321 }
 322
 323 void Parser::onClassConstant(Token &out, Token *exprs, Token &var,
 324                              Token &value) {
 325   onVariable(out, exprs, var, &value, true, m_scanner.detachDocComment());
 326 }
 327
 328 void Parser::onVariable(Token &out, Token *exprs, Token &var, Token *value,
 329                         bool constant /* = false */,
 330                         const std::string &docComment /* = "" */) {
 331   ExpressionPtr expList;
 332   if (exprs) {
 333     expList = exprs->exp;
 334   } else {
 335     expList = NEW_EXP0(ExpressionList);
 336   }
 337   ExpressionPtr exp;
 338   if (constant) {
 339     exp = NEW_EXP(ConstantExpression, var->text(), false, docComment);
 340   } else {
 341     exp = NEW_EXP(SimpleVariable, var->text(), docComment);
 342   }
 343   if (value) {
 344     exp = NEW_EXP(AssignmentExpression, exp, value->exp, false);
 345   }
 346   expList->addElement(exp);
 347   out->exp = expList;
 348 }
 349
 350 void Parser::onSimpleVariable(Token &out, Token &var) {
 351   out->exp = NEW_EXP(SimpleVariable, var->text());
 352 }
 353
 354 void Parser::onDynamicVariable(Token &out, Token &expr, bool encap) {
 355   out->exp = getDynamicVariable(expr->exp, encap);
 356 }
 357
 358 void Parser::onIndirectRef(Token &out, Token &refCount, Token &var) {
 359   out->exp = var->exp;
 360   for (int i = 0; i < refCount->num(); i++) {
 361     out->exp = createDynamicVariable(out->exp);
 362   }
 363 }
 364
 365 void Parser::onStaticMember(Token &out, Token &cls, Token &name) {
 366   if (name->exp->is(Expression::KindOfArrayElementExpression) &&
 367       dynamic_pointer_cast<ArrayElementExpression>(name->exp)->
 368       appendClass(cls->exp, m_ar, m_file)) {
 369     out->exp = name->exp;
 370   } else {
 371     StaticMemberExpressionPtr sme = NEW_EXP(StaticMemberExpression,
 372                                             cls->exp, name->exp);
 373     sme->onParse(m_ar, m_file);
 374     out->exp = sme;
 375   }
 376 }
 377
 378 void Parser::onRefDim(Token &out, Token &var, Token &offset) {
 379   if (!var->exp) {
 380     var->exp = NEW_EXP(ConstantExpression, var->text(), var->num() & 2);
 381   }
 382   if (!offset->exp) {
 383     UnaryOpExpressionPtr uop;
 384
 385     if (dynamic_pointer_cast<FunctionCall>(var->exp)) {
 386       PARSE_ERROR("Can't use function call result as array base"
 387                      " in write context");
 388     } else if ((uop = dynamic_pointer_cast<UnaryOpExpression>(var->exp))
 389                && uop->getOp() == T_ARRAY) {
 390       PARSE_ERROR("Can't use array() as base in write context");
 391     }
 392   }
 393   out->exp = NEW_EXP(ArrayElementExpression, var->exp, offset->exp);
 394 }
 395
 396 ExpressionPtr Parser::getDynamicVariable(ExpressionPtr exp, bool encap) {
 397   if (encap) {
 398     ConstantExpressionPtr var = dynamic_pointer_cast<ConstantExpression>(exp);
 399     if (var) {
 400       return NEW_EXP(SimpleVariable, var->getName());
 401     }
 402   } else {
 403     ScalarExpressionPtr var = dynamic_pointer_cast<ScalarExpression>(exp);
 404     if (var) {
 405       return NEW_EXP(SimpleVariable, var->getString());
 406     }
 407   }
 408   return createDynamicVariable(exp);
 409 }
 410
 411 ExpressionPtr Parser::createDynamicVariable(ExpressionPtr exp) {
 412   m_file->setAttribute(FileScope::ContainsDynamicVariable);
 413   return NEW_EXP(DynamicVariable, exp);
 414 }
 415
 416 void Parser::onCallParam(Token &out, Token *params, Token &expr, bool ref) {
 417   if (!params) {
 418     out->exp = NEW_EXP0(ExpressionList);
 419   } else {
 420     out->exp = params->exp;
 421   }
 422   if (ref) {
 423     expr->exp->setContext(Expression::RefParameter);
 424     expr->exp->setContext(Expression::RefValue);
 425   }
 426   out->exp->addElement(expr->exp);
 427 }
 428
 429 void Parser::onCall(Token &out, bool dynamic, Token &name, Token &params,
 430                     Token *cls) {
 431   ExpressionPtr clsExp;
 432   if (cls) {
 433     clsExp = cls->exp;
 434   }
 435   if (dynamic) {
 436     out->exp = NEW_EXP(DynamicFunctionCall, name->exp,
 437                        dynamic_pointer_cast<ExpressionList>(params->exp),
 438                        clsExp);
 439   } else {
 440     string funcName = name.text();
 441     // strip out namespaces for func_get_args and friends check
 442     size_t lastBackslash = funcName.find_last_of(NAMESPACE_SEP);
 443     const string stripped = lastBackslash == string::npos
 444                             ? funcName
 445                             : funcName.substr(lastBackslash+1);
 446     bool hadBackslash = name->num() & 2;
 447
 448     if (!cls && !hadBackslash) {
 449       if (stripped == "func_num_args" ||
 450           stripped == "func_get_args" ||
 451           stripped == "func_get_arg") {
 452         funcName = stripped;
 453         if (m_hasCallToGetArgs.size() > 0) {
 454           m_hasCallToGetArgs.back() = true;
 455         }
 456       }
 457       // Auto import a few functions from the HH namespace
 458       if (isAutoAliasOn() &&
 459           (stripped == "fun" ||
 460            stripped == "meth_caller" ||
 461            stripped == "class_meth" ||
 462            stripped == "inst_meth" ||
 463            stripped == "invariant_callback_register" ||
 464            stripped == "invariant" ||
 465            stripped == "invariant_violation" ||
 466            stripped == "tuple"
 467           )) {
 468         funcName = "HH\\" + stripped;
 469       }
 470     }
 471
 472     SimpleFunctionCallPtr call
 473       (new RealSimpleFunctionCall
 474        (BlockScopePtr(), getLocation(),
 475         funcName, hadBackslash,
 476         dynamic_pointer_cast<ExpressionList>(params->exp), clsExp));
 477     if (m_scanner.isHHSyntaxEnabled() && !(name->num() & 2)) {
 478       // If the function name is without any backslashes or
 479       // namespace qualification then we treat this as a candidate
 480       // for optimization via bytecode promotion.
 481       // "idx" is the only function in that class for now but it's
 482       // cheaper to set the bit that to check for the function name
 483       call->setOptimizable();
 484     }
 485     out->exp = call;
 486
 487     call->onParse(m_ar, m_file);
 488   }
 489 }
 490
 491 ///////////////////////////////////////////////////////////////////////////////
 492 // object property and method calls
 493
 494 void Parser::onObjectProperty(Token &out, Token &base, Token &prop) {
 495   if (!prop->exp) {
 496     prop->exp = NEW_EXP(ScalarExpression, T_STRING, prop->text());
 497   }
 498   out->exp = NEW_EXP(ObjectPropertyExpression, base->exp, prop->exp);
 499 }
 500
 501 void Parser::onObjectMethodCall(Token &out, Token &base, Token &prop,
 502                                 Token &params) {
 503   if (!prop->exp) {
 504     prop->exp = NEW_EXP(ScalarExpression, T_STRING, prop->text());
 505   }
 506   ExpressionListPtr paramsExp;
 507   if (params->exp) {
 508     paramsExp = dynamic_pointer_cast<ExpressionList>(params->exp);
 509   } else {
 510     paramsExp = NEW_EXP0(ExpressionList);
 511   }
 512   out->exp = NEW_EXP(ObjectMethodExpression, base->exp, prop->exp, paramsExp);
 513 }
 514
 515 ///////////////////////////////////////////////////////////////////////////////
 516 // encapsed expressions
 517
 518 void Parser::onEncapsList(Token &out, int type, Token &list) {
 519   out->exp = NEW_EXP(EncapsListExpression, type,
 520                      dynamic_pointer_cast<ExpressionList>(list->exp));
 521 }
 522
 523 void Parser::addEncap(Token &out, Token *list, Token &expr, int type) {
 524   ExpressionListPtr expList;
 525   if (list && list->exp) {
 526     expList = dynamic_pointer_cast<ExpressionList>(list->exp);
 527   } else {
 528     expList = NEW_EXP0(ExpressionList);
 529   }
 530   ExpressionPtr exp;
 531   if (type == -1) {
 532     exp = expr->exp;
 533   } else {
 534     exp = NEW_EXP(ScalarExpression, T_ENCAPSED_AND_WHITESPACE,
 535                   expr->text(), true);
 536   }
 537   expList->addElement(exp);
 538   out->exp = expList;
 539 }
 540
 541 void Parser::encapRefDim(Token &out, Token &var, Token &offset) {
 542   ExpressionPtr dim;
 543   switch (offset->num()) {
 544   case T_STRING:
 545     dim = NEW_EXP(ScalarExpression, T_STRING, offset->text(), true);
 546     break;
 547   case T_NUM_STRING:
 548     dim = NEW_EXP(ScalarExpression, T_NUM_STRING, offset->text());
 549     break;
 550   case T_VARIABLE:
 551     dim = NEW_EXP(SimpleVariable, offset->text());
 552     break;
 553   default:
 554     assert(false);
 555   }
 556
 557   ExpressionPtr arr = NEW_EXP(SimpleVariable, var->text());
 558   out->exp = NEW_EXP(ArrayElementExpression, arr, dim);
 559 }
 560
 561 void Parser::encapObjProp(Token &out, Token &var, Token &name) {
 562   ExpressionPtr obj = NEW_EXP(SimpleVariable, var->text());
 563
 564   ExpressionPtr prop = NEW_EXP(ScalarExpression, T_STRING, name->text());
 565   out->exp = NEW_EXP(ObjectPropertyExpression, obj, prop);
 566 }
 567
 568 void Parser::encapArray(Token &out, Token &var, Token &expr) {
 569   ExpressionPtr arr = NEW_EXP(SimpleVariable, var->text());
 570   out->exp = NEW_EXP(ArrayElementExpression, arr, expr->exp);
 571 }
 572
 573 ///////////////////////////////////////////////////////////////////////////////
 574 // expressions
 575
 576 void Parser::onConstantValue(Token &out, Token &constant) {
 577   ConstantExpressionPtr con = NEW_EXP(ConstantExpression, constant->text(),
 578       constant->num() & 2);
 579   con->onParse(m_ar, m_file);
 580   out->exp = con;
 581 }
 582
 583 void Parser::onScalar(Token &out, int type, Token &scalar) {
 584   if (type == T_FILE || type == T_DIR) {
 585     onUnaryOpExp(out, scalar, type, true);
 586     return;
 587   }
 588
 589   ScalarExpressionPtr exp;
 590   switch (type) {
 591     case T_METHOD_C:
 592       if (m_inTrait) {
 593         exp = NEW_EXP(ScalarExpression, type, scalar->text(),
 594                       m_clsName + "::" + m_funcName);
 595       } else {
 596         exp = NEW_EXP(ScalarExpression, type, scalar->text());
 597       }
 598       break;
 599     case T_STRING:
 600     case T_LNUMBER:
 601     case T_DNUMBER:
 602     case T_ONUMBER:
 603     case T_LINE:
 604     case T_COMPILER_HALT_OFFSET:
 605     case T_FUNC_C:
 606     case T_CLASS_C:
 607       exp = NEW_EXP(ScalarExpression, type, scalar->text());
 608       break;
 609     case T_TRAIT_C:
 610       exp = NEW_EXP(ScalarExpression, type, scalar->text(),
 611                     m_inTrait ? m_clsName : "");
 612       break;
 613     case T_NS_C:
 614       exp = NEW_EXP(ScalarExpression, type, m_namespace);
 615       break;
 616     case T_CONSTANT_ENCAPSED_STRING:
 617       exp = NEW_EXP(ScalarExpression, type, scalar->text(), true);
 618       break;
 619     default:
 620       assert(false);
 621   }
 622   if (type == T_COMPILER_HALT_OFFSET) {
 623     // Keep track of this expression for later backpatching
 624     // If it doesn't get backpatched (because there was no HALT_COMPILER
 625     // then the constant will return (int)"__COMPILER_HALT_OFFSET__" (zero)
 626     m_compilerHaltOffsetVec.push_back(exp);
 627   }
 628   out->exp = exp;
 629 }
 630
 631 void Parser::onExprListElem(Token &out, Token *exprs, Token &expr) {
 632   ExpressionPtr expList;
 633   if (exprs && exprs->exp) {
 634     expList = exprs->exp;
 635   } else {
 636     expList = NEW_EXP0(ExpressionList);
 637   }
 638   expList->addElement(expr->exp);
 639   out->exp = expList;
 640 }
 641
 642 void Parser::onListAssignment(Token &out, Token &vars, Token *expr,
 643                               bool rhsFirst /* = false */) {
 644   ExpressionListPtr el(dynamic_pointer_cast<ExpressionList>(vars->exp));
 645   for (int i = 0; i < el->getCount(); i++) {
 646     if (dynamic_pointer_cast<FunctionCall>((*el)[i])) {
 647       PARSE_ERROR("Can't use return value in write context");
 648     }
 649   }
 650   out->exp = NEW_EXP(ListAssignment,
 651                      dynamic_pointer_cast<ExpressionList>(vars->exp),
 652                      expr ? expr->exp : ExpressionPtr(), rhsFirst);
 653 }
 654
 655 void Parser::onAListVar(Token &out, Token *list, Token *var) {
 656   Token empty_list, empty_var;
 657   if (!var) {
 658     empty_var.exp = ExpressionPtr();
 659     var = &empty_var;
 660   }
 661   if (!list) {
 662     empty_list.exp = NEW_EXP0(ExpressionList);
 663     list = &empty_list;
 664   }
 665   onExprListElem(out, list, *var);
 666 }
 667
 668 void Parser::onAListSub(Token &out, Token *list, Token &sublist) {
 669   onListAssignment(out, sublist, nullptr);
 670   onExprListElem(out, list, out);
 671 }
 672
 673 void Parser::checkAssignThis(Token &var) {
 674   if (SimpleVariablePtr simp = dynamic_pointer_cast<SimpleVariable>(var.exp)) {
 675     if (simp->getName() == "this") {
 676       PARSE_ERROR("Cannot re-assign $this");
 677     }
 678   }
 679 }
 680
 681 void Parser::onAssign(Token &out, Token &var, Token &expr, bool ref,
 682                       bool rhsFirst /* = false */) {
 683   if (dynamic_pointer_cast<FunctionCall>(var->exp)) {
 684     PARSE_ERROR("Can't use return value in write context");
 685   }
 686   checkAssignThis(var);
 687   out->exp = NEW_EXP(AssignmentExpression, var->exp, expr->exp, ref, rhsFirst);
 688 }
 689
 690 void Parser::onAssignNew(Token &out, Token &var, Token &name, Token &args) {
 691   checkAssignThis(var);
 692   ExpressionPtr exp =
 693     NEW_EXP(NewObjectExpression, name->exp,
 694             dynamic_pointer_cast<ExpressionList>(args->exp));
 695   out->exp = NEW_EXP(AssignmentExpression, var->exp, exp, true);
 696 }
 697
 698 void Parser::onNewObject(Token &out, Token &name, Token &args) {
 699   NewObjectExpressionPtr new_obj =
 700     NEW_EXP(NewObjectExpression, name->exp,
 701             dynamic_pointer_cast<ExpressionList>(args->exp));
 702   new_obj->onParse(m_ar, m_file);
 703   out->exp = new_obj;
 704 }
 705
 706 void Parser::onUnaryOpExp(Token &out, Token &operand, int op, bool front) {
 707   switch (op) {
 708   case T_INCLUDE:
 709   case T_INCLUDE_ONCE:
 710   case T_REQUIRE:
 711   case T_REQUIRE_ONCE:
 712     {
 713       IncludeExpressionPtr exp = NEW_EXP(IncludeExpression, operand->exp, op);
 714       out->exp = exp;
 715       exp->onParse(m_ar, m_file);
 716     }
 717     break;
 718   case T_INC:
 719   case T_DEC:
 720   case T_ISSET:
 721   case T_UNSET:
 722     if (dynamic_pointer_cast<FunctionCall>(operand->exp)) {
 723       PARSE_ERROR("Can't use return value in write context");
 724     }
 725   default:
 726     {
 727       UnaryOpExpressionPtr exp = NEW_EXP(UnaryOpExpression, operand->exp, op,
 728                                          front);
 729       out->exp = exp;
 730       exp->onParse(m_ar, m_file);
 731     }
 732     break;
 733   }
 734 }
 735
 736 void Parser::onBinaryOpExp(Token &out, Token &operand1, Token &operand2,
 737                            int op) {
 738   BinaryOpExpressionPtr bop =
 739     NEW_EXP(BinaryOpExpression, operand1->exp, operand2->exp, op);
 740
 741   if (bop->isAssignmentOp() &&
 742       dynamic_pointer_cast<FunctionCall>(operand1->exp)) {
 743     PARSE_ERROR("Can't use return value in write context");
 744   }
 745
 746   out->exp = bop;
 747
 748   // If the operands are simple enough we can fold this expression right
 749   // here and keep the parse tree smaller.
 750   if (ExpressionPtr optExp = bop->foldConst(m_ar)) out->exp = optExp;
 751 }
 752
 753 void Parser::onQOp(Token &out, Token &exprCond, Token *expYes, Token &expNo) {
 754   out->exp = NEW_EXP(QOpExpression, exprCond->exp,
 755                      expYes ? expYes->exp : ExpressionPtr(), expNo->exp);
 756 }
 757
 758 void Parser::onArray(Token &out, Token &pairs, int op /* = T_ARRAY */) {
 759   if (op != T_ARRAY && !m_scanner.isHHSyntaxEnabled()) {
 760     PARSE_ERROR("Typed collection is not enabled");
 761     return;
 762   }
 763   onUnaryOpExp(out, pairs, T_ARRAY, true);
 764 }
 765
 766 void Parser::onArrayPair(Token &out, Token *pairs, Token *name, Token &value,
 767                          bool ref) {
 768   if (!value->exp) return;
 769
 770   ExpressionPtr expList;
 771   if (pairs && pairs->exp) {
 772     expList = pairs->exp;
 773   } else {
 774     expList = NEW_EXP0(ExpressionList);
 775   }
 776   ExpressionPtr nameExp = name ? name->exp : ExpressionPtr();
 777   expList->addElement(NEW_EXP(ArrayPairExpression, nameExp, value->exp, ref));
 778   out->exp = expList;
 779 }
 780
 781 void Parser::onEmptyCollection(Token &out) {
 782   out->exp = NEW_EXP0(ExpressionList);
 783 }
 784
 785 void
 786 Parser::onCollectionPair(Token &out, Token *pairs, Token *name, Token &value) {
 787   if (!value->exp) return;
 788
 789   ExpressionPtr expList;
 790   if (pairs && pairs->exp) {
 791     expList = pairs->exp;
 792   } else {
 793     expList = NEW_EXP0(ExpressionList);
 794   }
 795   ExpressionPtr nameExp = name ? name->exp : ExpressionPtr();
 796   expList->addElement(NEW_EXP(ArrayPairExpression, nameExp, value->exp, false,
 797                               true));
 798   out->exp = expList;
 799 }
 800
 801 void Parser::onUserAttribute(Token &out, Token *attrList, Token &name,
 802                              Token &value) {
 803   ExpressionPtr expList;
 804   if (attrList && attrList->exp) {
 805     expList = attrList->exp;
 806   } else {
 807     expList = NEW_EXP0(ExpressionList);
 808   }
 809   expList->addElement(NEW_EXP(UserAttribute, name->text(), value->exp));
 810   out->exp = expList;
 811 }
 812
 813 void Parser::onClassConst(Token &out, Token &cls, Token &name, bool text) {
 814   if (!cls->exp) {
 815     cls->exp = NEW_EXP(ScalarExpression, T_STRING, cls->text());
 816   }
 817   ClassConstantExpressionPtr con =
 818     NEW_EXP(ClassConstantExpression, cls->exp, name->text());
 819   con->onParse(m_ar, m_file);
 820   out->exp = con;
 821 }
 822
 823 void Parser::onClassClass(Token &out, Token &cls, Token &name,
 824                           bool inStaticContext) {
 825   if (inStaticContext) {
 826     if (cls->same("parent") || cls->same("static")) {
 827       PARSE_ERROR(
 828         "%s::class cannot be used for compile-time class name resolution",
 829         cls->text().c_str()
 830       );
 831       return;
 832     }
 833   }
 834   if (cls->exp && !cls->exp->is(Expression::KindOfScalarExpression)) {
 835     PARSE_ERROR("::class can only be used on scalars");
 836   }
 837   if (cls->same("self") || cls->same("parent") || cls->same("static")) {
 838     if (cls->same("self") && m_inTrait) {
 839       // Sooo... self:: works dynamically for everything in a trait except
 840       // for self::CLASS where it returns the trait name. Great...
 841       onScalar(out, T_TRAIT_C, cls);
 842     } else {
 843       onClassConst(out, cls, name, inStaticContext);
 844     }
 845   } else {
 846     onScalar(out, T_STRING, cls);
 847   }
 848 }
 849
 850 ///////////////////////////////////////////////////////////////////////////////
 851 // function/method declaration
 852
 853 void Parser::onFunctionStart(Token &name, bool doPushComment /* = true */) {
 854   m_file->pushAttribute();
 855   if (doPushComment) {
 856     pushComment();
 857   }
 858   newScope();
 859   m_funcContexts.push_back(FunctionContext());
 860   m_funcName = name.text();
 861   m_hasCallToGetArgs.push_back(false);
 862   m_staticVars.push_back(StringToExpressionPtrVecMap());
 863 }
 864
 865 void Parser::onMethodStart(Token &name, Token &mods,
 866                            bool doPushComment /* = true */) {
 867   onFunctionStart(name, doPushComment);
 868 }
 869
 870 void Parser::fixStaticVars() {
 871   StringToExpressionPtrVecMap &m = m_staticVars.back();
 872   for (StringToExpressionPtrVecMap::iterator it = m.begin(), end = m.end();
 873        it != end; ++it) {
 874     const ExpressionPtrVec &v = it->second;
 875     if (v.size() > 1) {
 876       ExpressionPtr last;
 877       for (int i = v.size(); i--; ) {
 878         ExpressionListPtr el(dynamic_pointer_cast<ExpressionList>(v[i]));
 879         for (int j = el->getCount(); j--; ) {
 880           ExpressionPtr s = (*el)[j];
 881           SimpleVariablePtr v = dynamic_pointer_cast<SimpleVariable>(
 882             s->is(Expression::KindOfAssignmentExpression) ?
 883             static_pointer_cast<AssignmentExpression>(s)->getVariable() : s);
 884           if (v->getName() == it->first) {
 885             if (!last) {
 886               last = s;
 887             } else {
 888               el->removeElement(j);
 889               el->insertElement(last->clone(), j);
 890             }
 891           }
 892         }
 893       }
 894     }
 895   }
 896   m_staticVars.pop_back();
 897 }
 898
 899 void Parser::checkFunctionContext(string funcName,
 900                                   FunctionContext& funcContext,
 901                                   ModifierExpressionPtr modifiers,
 902                                   int returnsRef) {
 903   funcContext.checkFinalAssertions();
 904
 905   // let async modifier be mandatory
 906   if (funcContext.isAsync && !modifiers->isAsync()) {
 907     PARSE_ERROR("Function '%s' contains 'await' but is not declared as async.",
 908                 funcName.c_str());
 909   }
 910
 911   if (modifiers->isAsync() && returnsRef) {
 912     PARSE_ERROR("Asynchronous function '%s' cannot return reference.",
 913                 funcName.c_str());
 914   }
 915
 916   if (modifiers->isAsync() && funcContext.isGenerator) {
 917     PARSE_ERROR("'yield' is not allowed in async functions.");
 918   }
 919 }
 920
 921 void Parser::prepareConstructorParameters(StatementListPtr stmts,
 922                                           ExpressionListPtr params,
 923                                           bool isAbstract) {
 924   for (int i = 0, count = params->getCount(); i < count; i++) {
 925     ParameterExpressionPtr param =
 926         dynamic_pointer_cast<ParameterExpression>((*params)[i]);
 927     TokenID mod = param->getModifier();
 928     if (mod == 0) continue;
 929
 930     if (isAbstract) {
 931        param->parseTimeFatal(Compiler::InvalidAttribute,
 932                              "parameter modifiers not allowed on "
 933                              "abstract __construct");
 934     }
 935     if (!stmts) {
 936        param->parseTimeFatal(Compiler::InvalidAttribute,
 937                              "parameter modifiers not allowed on "
 938                              "__construct without a body");
 939     }
 940     if (param->annotation()) {
 941       std::vector<std::string> typeNames;
 942       param->annotation()->getAllSimpleNames(typeNames);
 943       for (auto& typeName : typeNames) {
 944         if (isTypeVarInImmediateScope(typeName)) {
 945           param->parseTimeFatal(Compiler::InvalidAttribute,
 946                                 "parameter modifiers not supported with "
 947                                 "type variable annotation");
 948         }
 949       }
 950     }
 951     std::string name = param->getName();
 952     SimpleVariablePtr value = NEW_EXP(SimpleVariable, name);
 953     ScalarExpressionPtr prop = NEW_EXP(ScalarExpression, T_STRING, name);
 954     SimpleVariablePtr self = NEW_EXP(SimpleVariable, "this");
 955     ObjectPropertyExpressionPtr objProp =
 956         NEW_EXP(ObjectPropertyExpression, self, prop);
 957     AssignmentExpressionPtr assign =
 958         NEW_EXP(AssignmentExpression, objProp, value, false);
 959     ExpStatementPtr stmt = NEW_STMT(ExpStatement, assign);
 960     stmts->insertElement(stmt);
 961   }
 962 }
 963
 964 string Parser::getFunctionName(FunctionType type, Token* name) {
 965   switch (type) {
 966     case FunctionType::Closure:
 967       return newClosureName(m_clsName, m_containingFuncName);
 968     case FunctionType::Function:
 969       assert(name);
 970       if (!m_lambdaMode) {
 971         return name->text();
 972       } else {
 973         return name->text() + "{lambda}";
 974       }
 975     case FunctionType::Method:
 976       assert(name);
 977       return name->text();
 978   }
 979   not_reached();
 980 }
 981
 982 StatementPtr Parser::onFunctionHelper(FunctionType type,
 983                               Token *modifiers, Token &ret,
 984                               Token &ref, Token *name, Token &params,
 985                               Token &stmt, Token *attr, bool reloc) {
 986   // prepare and validate function modifiers
 987   ModifierExpressionPtr modifiersExp = modifiers && modifiers->exp ?
 988     dynamic_pointer_cast<ModifierExpression>(modifiers->exp)
 989     : NEW_EXP0(ModifierExpression);
 990   modifiersExp->setHasPrivacy(type == FunctionType::Method);
 991   if (type == FunctionType::Closure && !modifiersExp->validForClosure()) {
 992     PARSE_ERROR("Invalid modifier on closure funciton.");
 993   }
 994   if (type == FunctionType::Function && !modifiersExp->validForFunction()) {
 995     PARSE_ERROR("Invalid modifier on function %s.", name->text().c_str());
 996   }
 997
 998   StatementListPtr stmts = stmt->stmt || stmt->num() != 1 ?
 999     dynamic_pointer_cast<StatementList>(stmt->stmt)
1000     : NEW_STMT0(StatementList);
1001
1002   ExpressionListPtr old_params =
1003     dynamic_pointer_cast<ExpressionList>(params->exp);
1004
1005   string funcName = getFunctionName(type, name);
1006
1007   if (type == FunctionType::Method && old_params &&
1008       funcName == "__construct") {
1009     prepareConstructorParameters(stmts, old_params,
1010                                  modifiersExp->isAbstract());
1011   }
1012
1013   fixStaticVars();
1014
1015   int attribute = m_file->popAttribute();
1016   string comment = popComment();
1017
1018   ExpressionListPtr attrList;
1019   if (attr && attr->exp) {
1020     attrList = dynamic_pointer_cast<ExpressionList>(attr->exp);
1021   }
1022
1023   // create function/method statement
1024   FunctionStatementPtr func;
1025   MethodStatementPtr mth;
1026   if (type == FunctionType::Method) {
1027     mth = NEW_STMT(MethodStatement, modifiersExp,
1028                    ref->num(), funcName, old_params,
1029                    ret.typeAnnotation, stmts,
1030                    attribute, comment, attrList);
1031     completeScope(mth->onInitialParse(m_ar, m_file));
1032   } else {
1033     func = NEW_STMT(FunctionStatement, modifiersExp,
1034                    ref->num(), funcName, old_params,
1035                    ret.typeAnnotation, stmts,
1036                    attribute, comment, attrList);
1037
1038     func->onParse(m_ar, m_file);
1039     completeScope(func->getFunctionScope());
1040     if (func->ignored()) {
1041       return NEW_STMT0(StatementList);
1042     }
1043     mth = func;
1044   }
1045
1046   // check and set generator/async flags
1047   FunctionContext funcContext = m_funcContexts.back();
1048   checkFunctionContext(funcName, funcContext, modifiersExp, ref->num());
1049   mth->getFunctionScope()->setGenerator(funcContext.isGenerator);
1050   mth->getFunctionScope()->setAsync(modifiersExp->isAsync());
1051   m_funcContexts.pop_back();
1052
1053   mth->setHasCallToGetArgs(m_hasCallToGetArgs.back());
1054   m_hasCallToGetArgs.pop_back();
1055
1056   LocationPtr loc = popFuncLocation();
1057   if (reloc) {
1058     mth->getLocation()->line0 = loc->line0;
1059     mth->getLocation()->char0 = loc->char0;
1060   }
1061
1062   return mth;
1063 }
1064
1065 void Parser::onFunction(Token &out, Token *modifiers, Token &ret, Token &ref,
1066                         Token &name, Token &params, Token &stmt, Token *attr) {
1067   out->stmt = onFunctionHelper(FunctionType::Function,
1068                 modifiers, ret, ref, &name, params, stmt, attr, true);
1069 }
1070
1071 void Parser::onMethod(Token &out, Token &modifiers, Token &ret, Token &ref,
1072                       Token &name, Token &params, Token &stmt,
1073                       Token *attr, bool reloc /* = true */) {
1074   out->stmt = onFunctionHelper(FunctionType::Method,
1075                 &modifiers, ret, ref, &name, params, stmt, attr, reloc);
1076 }
1077
1078 void Parser::onVariadicParam(Token &out, Token *params,
1079                              Token &type, Token &var,
1080                              bool ref, Token *attr, Token *modifier) {
1081 }
1082
1083 void Parser::onParam(Token &out, Token *params, Token &type, Token &var,
1084                      bool ref, Token *defValue, Token *attr, Token *modifier) {
1085   ExpressionPtr expList;
1086   if (params) {
1087     expList = params->exp;
1088   } else {
1089     expList = NEW_EXP0(ExpressionList);
1090   }
1091   ExpressionListPtr attrList;
1092   if (attr && attr->exp) {
1093     attrList = dynamic_pointer_cast<ExpressionList>(attr->exp);
1094   }
1095
1096   TypeAnnotationPtr typeAnnotation = type.typeAnnotation;
1097   expList->addElement(NEW_EXP(ParameterExpression, typeAnnotation,
1098                               m_scanner.isHHSyntaxEnabled(), var->text(),
1099                               ref, (modifier) ? modifier->num() : 0,
1100                               defValue ? defValue->exp : ExpressionPtr(),
1101                               attrList));
1102   out->exp = expList;
1103 }
1104
1105 void Parser::onClassStart(int type, Token &name) {
1106   const Type::TypePtrMap& typeHintTypes =
1107     Type::GetTypeHintTypes(m_scanner.isHHSyntaxEnabled());
1108   if (0 == strcasecmp("self", name.text().c_str()) ||
1109       0 == strcasecmp("parent", name.text().c_str()) ||
1110       typeHintTypes.find(name.text()) != typeHintTypes.end()) {
1111     PARSE_ERROR("Cannot use '%s' as class name as it is reserved",
1112                 name.text().c_str());
1113   }
1114
1115   pushComment();
1116   newScope();
1117   m_clsName = name.text();
1118   m_inTrait = type == T_TRAIT;
1119 }
1120
1121 void Parser::onClass(Token &out, int type, Token &name, Token &base,
1122                      Token &baseInterface, Token &stmt, Token *attr) {
1123   StatementListPtr stmtList;
1124   if (stmt->stmt) {
1125     stmtList = dynamic_pointer_cast<StatementList>(stmt->stmt);
1126   }
1127   ExpressionListPtr attrList;
1128   if (attr && attr->exp) {
1129     attrList = dynamic_pointer_cast<ExpressionList>(attr->exp);
1130   }
1131
1132   ClassStatementPtr cls = NEW_STMT
1133     (ClassStatement, type, name->text(), base->text(),
1134      dynamic_pointer_cast<ExpressionList>(baseInterface->exp),
1135      popComment(), stmtList, attrList);
1136
1137   // look for argument promotion in ctor
1138   ExpressionListPtr promote = NEW_EXP(ExpressionList);
1139   cls->checkArgumentsToPromote(promote, type);
1140   auto count = promote->getCount();
1141   cls->setPromotedParameterCount(count);
1142   for (int i = 0; i < count; i++) {
1143     auto param =
1144         dynamic_pointer_cast<ParameterExpression>((*promote)[i]);
1145     TokenID mod = param->getModifier();
1146     std::string name = param->getName();
1147     std::string type = param->hasUserType() ?
1148                                   param->getUserTypeHint() : "";
1149
1150     // create the class variable and change the location to
1151     // point to the parameter location for error reporting
1152     LocationPtr location = param->getLocation();
1153     ModifierExpressionPtr modifier = NEW_EXP0(ModifierExpression);
1154     modifier->add(mod);
1155     modifier->setLocation(location);
1156     SimpleVariablePtr svar = NEW_EXP(SimpleVariable, name);
1157     svar->setLocation(location);
1158     ExpressionListPtr expList = NEW_EXP0(ExpressionList);
1159     expList->addElement(svar);
1160     expList->setLocation(location);
1161     ClassVariablePtr var = NEW_STMT(ClassVariable, modifier, type, expList);
1162     var->setLocation(location);
1163     cls->getStmts()->addElement(var);
1164   }
1165
1166   out->stmt = cls;
1167   {
1168     cls->onParse(m_ar, m_file);
1169   }
1170   completeScope(cls->getClassScope());
1171   if (cls->ignored()) {
1172     out->stmt = NEW_STMT0(StatementList);
1173   }
1174   m_clsName.clear();
1175   m_inTrait = false;
1176   registerAlias(name.text());
1177 }
1178
1179 void Parser::onInterface(Token &out, Token &name, Token &base, Token &stmt,
1180                          Token *attr) {
1181   StatementListPtr stmtList;
1182   if (stmt->stmt) {
1183     stmtList = dynamic_pointer_cast<StatementList>(stmt->stmt);
1184   }
1185   ExpressionListPtr attrList;
1186   if (attr && attr->exp) {
1187     attrList = dynamic_pointer_cast<ExpressionList>(attr->exp);
1188   }
1189
1190   InterfaceStatementPtr intf = NEW_STMT
1191     (InterfaceStatement, name->text(),
1192      dynamic_pointer_cast<ExpressionList>(base->exp), popComment(), stmtList,
1193      attrList);
1194   out->stmt = intf;
1195   {
1196     intf->onParse(m_ar, m_file);
1197   }
1198   completeScope(intf->getClassScope());
1199 }
1200
1201 void Parser::onInterfaceName(Token &out, Token *names, Token &name) {
1202   ExpressionPtr expList;
1203   if (names) {
1204     expList = names->exp;
1205   } else {
1206     expList = NEW_EXP0(ExpressionList);
1207   }
1208   expList->addElement(NEW_EXP(ScalarExpression, T_STRING, name->text()));
1209   out->exp = expList;
1210 }
1211
1212 void Parser::onTraitRequire(Token &out, Token &name, bool isExtends) {
1213   out->stmt = NEW_STMT(TraitRequireStatement, name->text(), isExtends);
1214 }
1215
1216 void Parser::onTraitUse(Token &out, Token &traits, Token &rules) {
1217   if (!rules->stmt) {
1218     rules->stmt = NEW_STMT0(StatementList);
1219   }
1220   out->stmt = NEW_STMT(UseTraitStatement,
1221                        dynamic_pointer_cast<ExpressionList>(traits->exp),
1222                        dynamic_pointer_cast<StatementList>(rules->stmt));
1223 }
1224
1225 void Parser::onTraitName(Token &out, Token *names, Token &name) {
1226   ExpressionPtr expList;
1227   if (names) {
1228     expList = names->exp;
1229   } else {
1230     expList = NEW_EXP0(ExpressionList);
1231   }
1232   expList->addElement(NEW_EXP(ScalarExpression, T_STRING, name->text()));
1233   out->exp = expList;
1234 }
1235
1236 void Parser::onTraitRule(Token &out, Token &stmtList, Token &newStmt) {
1237   if (!stmtList->stmt) {
1238     out->stmt = NEW_STMT0(StatementList);
1239   } else {
1240     out->stmt = stmtList->stmt;
1241   }
1242   assert(newStmt->stmt);
1243   out->stmt->addElement(newStmt->stmt);
1244 }
1245
1246 void Parser::onTraitPrecRule(Token &out, Token &traitName, Token &methodName,
1247                              Token &otherTraits) {
1248   assert(otherTraits->exp);
1249   ScalarExpressionPtr expTraitName = NEW_EXP(ScalarExpression, T_STRING,
1250                                              traitName->text());
1251   ScalarExpressionPtr expMethodName = NEW_EXP(ScalarExpression, T_STRING,
1252                                               methodName->text());
1253   out->stmt = NEW_STMT(TraitPrecStatement, expTraitName, expMethodName,
1254                        dynamic_pointer_cast<ExpressionList>(otherTraits->exp));
1255 }
1256
1257 void Parser::onTraitAliasRuleStart(Token &out, Token &traitName,
1258                                    Token &methodName) {
1259   ScalarExpressionPtr expTraitName = NEW_EXP(ScalarExpression, T_STRING,
1260                                              traitName->text());
1261   ScalarExpressionPtr expMethodName = NEW_EXP(ScalarExpression, T_STRING,
1262                                               methodName->text());
1263
1264   ModifierExpressionPtr expModifiers = NEW_EXP0(ModifierExpression);
1265
1266   ScalarExpressionPtr expNewMethodName = NEW_EXP(ScalarExpression, T_STRING,
1267                                                  methodName->text());
1268
1269   out->stmt = NEW_STMT(TraitAliasStatement, expTraitName, expMethodName,
1270                        expModifiers, expNewMethodName);
1271 }
1272
1273 void Parser::onTraitAliasRuleModify(Token &out, Token &rule,
1274                                     Token &accessModifiers,
1275                                     Token &newMethodName) {
1276   TraitAliasStatementPtr ruleStmt=
1277     dynamic_pointer_cast<TraitAliasStatement>(rule->stmt);
1278
1279   assert(ruleStmt);
1280
1281   if (!newMethodName->text().empty()) {
1282     ScalarExpressionPtr expNewMethodName =
1283       NEW_EXP(ScalarExpression, T_STRING, newMethodName->text());
1284     ruleStmt->setNewMethodName(expNewMethodName);
1285   }
1286
1287   if (accessModifiers->exp) {
1288     auto modifiersExp =
1289       dynamic_pointer_cast<ModifierExpression>(accessModifiers->exp);
1290     if (!modifiersExp->validForTraitAliasRule()) {
1291       PARSE_ERROR("Only access and visibility modifiers are allowed"
1292                   " in trait alias rule");
1293     }
1294     ruleStmt->setModifiers(modifiersExp);
1295   }
1296
1297   out->stmt = ruleStmt;
1298 }
1299
1300 void Parser::onClassVariableStart(Token &out, Token *modifiers, Token &decl,
1301                                   Token *type) {
1302   if (modifiers) {
1303     ModifierExpressionPtr exp = modifiers->exp ?
1304       dynamic_pointer_cast<ModifierExpression>(modifiers->exp)
1305       : NEW_EXP0(ModifierExpression);
1306
1307     out->stmt = NEW_STMT
1308       (ClassVariable, exp,
1309        (type) ? type->typeAnnotationName() : "",
1310        dynamic_pointer_cast<ExpressionList>(decl->exp));
1311   } else {
1312     out->stmt =
1313       NEW_STMT(ClassConstant,
1314         (type) ? type->typeAnnotationName() : "",
1315         dynamic_pointer_cast<ExpressionList>(decl->exp));
1316   }
1317 }
1318
1319 void Parser::onMemberModifier(Token &out, Token *modifiers, Token &modifier) {
1320   ModifierExpressionPtr expList;
1321   if (modifiers) {
1322     expList = dynamic_pointer_cast<ModifierExpression>(modifiers->exp);
1323   } else {
1324     expList = NEW_EXP0(ModifierExpression);
1325   }
1326   expList->add(modifier->num());
1327   out->exp = expList;
1328 }
1329
1330 ///////////////////////////////////////////////////////////////////////////////
1331 // statements
1332
1333 void Parser::initParseTree() {
1334   m_tree = NEW_STMT0(StatementList);
1335 }
1336
1337 void Parser::finiParseTree() {
1338   if (m_staticVars.size()) fixStaticVars();
1339   FunctionScopePtr pseudoMain = m_file->setTree(m_ar, m_tree);
1340   completeScope(pseudoMain);
1341   pseudoMain->setOuterScope(m_file);
1342   m_file->setOuterScope(m_ar);
1343   m_ar->parseExtraCode(m_file->getName());
1344   LocationPtr loc = getLocation();
1345   loc->line0 = loc->char0 = 1;
1346   pseudoMain->getStmt()->setLocation(loc);
1347 }
1348
1349 void Parser::onHaltCompiler() {
1350   if (m_nsState == InsideNamespace && !m_nsFileScope) {
1351     error("__HALT_COMPILER() can only be used from the outermost scope");
1352     return;
1353   }
1354   // Backpatch instances of __COMPILER_HALT_OFFSET__
1355   for(auto &cho : m_compilerHaltOffsetVec) {
1356      cho->setCompilerHaltOffset(m_scanner.getLocation()->cursor);
1357   }
1358 }
1359
1360 void Parser::onStatementListStart(Token &out) {
1361   out.reset();
1362 }
1363
1364 void Parser::addTopStatement(Token &new_stmt) {
1365   addStatement(m_tree, new_stmt->stmt);
1366 }
1367
1368 void Parser::addStatement(Token &out, Token &stmts, Token &new_stmt) {
1369   if (!stmts->stmt) {
1370     out->stmt = NEW_STMT0(StatementList);
1371   } else {
1372     out->stmt = stmts->stmt;
1373   }
1374   addStatement(out->stmt, new_stmt->stmt);
1375 }
1376
1377 void Parser::addStatement(StatementPtr stmt, StatementPtr new_stmt) {
1378   if (new_stmt) {
1379     stmt->addElement(new_stmt);
1380   }
1381 }
1382
1383 void Parser::finishStatement(Token &out, Token &stmts) {
1384   if (!stmts->stmt) {
1385     out->stmt = NEW_STMT0(StatementList);
1386   } else {
1387     out->stmt = stmts->stmt;
1388   }
1389 }
1390
1391 void Parser::onBlock(Token &out, Token &stmts) {
1392   if (!stmts->stmt) {
1393     stmts->stmt = NEW_STMT0(StatementList);
1394   } else if (!stmts->stmt->is(Statement::KindOfStatementList)) {
1395     out->stmt = NEW_STMT0(StatementList);
1396     out->stmt->addElement(stmts->stmt);
1397     stmts->stmt = out->stmt;
1398   }
1399   out->stmt = NEW_STMT(BlockStatement,
1400                        dynamic_pointer_cast<StatementList>(stmts->stmt));
1401 }
1402
1403 void Parser::onIf(Token &out, Token &cond, Token &stmt, Token &elseifs,
1404                   Token &elseStmt) {
1405   StatementPtr stmtList;
1406   if (!elseifs->stmt) {
1407     stmtList = NEW_STMT0(StatementList);
1408   } else {
1409     stmtList = elseifs->stmt;
1410   }
1411   if (stmt->stmt && stmt->stmt->is(Statement::KindOfStatementList)) {
1412     stmt->stmt = NEW_STMT(BlockStatement,
1413                           dynamic_pointer_cast<StatementList>(stmt->stmt));
1414   }
1415   stmtList->insertElement(NEW_STMT(IfBranchStatement, cond->exp, stmt->stmt));
1416   if (elseStmt->stmt) {
1417     if (elseStmt->stmt->is(Statement::KindOfStatementList)) {
1418       elseStmt->stmt = NEW_STMT
1419         (BlockStatement, dynamic_pointer_cast<StatementList>(elseStmt->stmt));
1420     }
1421     stmtList->addElement(NEW_STMT(IfBranchStatement, ExpressionPtr(),
1422                                   elseStmt->stmt));
1423   }
1424   out->stmt = NEW_STMT(IfStatement,
1425                        dynamic_pointer_cast<StatementList>(stmtList));
1426 }
1427
1428 void Parser::onElseIf(Token &out, Token &elseifs, Token &cond, Token &stmt) {
1429   if (!elseifs->stmt) {
1430     out->stmt = NEW_STMT0(StatementList);
1431   } else {
1432     out->stmt = elseifs->stmt;
1433   }
1434   if (stmt->stmt && stmt->stmt->is(Statement::KindOfStatementList)) {
1435     stmt->stmt = NEW_STMT(BlockStatement,
1436                           dynamic_pointer_cast<StatementList>(stmt->stmt));
1437   }
1438   out->stmt->addElement(NEW_STMT(IfBranchStatement, cond->exp, stmt->stmt));
1439 }
1440
1441 void Parser::onWhile(Token &out, Token &cond, Token &stmt) {
1442   if (stmt->stmt && stmt->stmt->is(Statement::KindOfStatementList)) {
1443     stmt->stmt = NEW_STMT(BlockStatement,
1444                           dynamic_pointer_cast<StatementList>(stmt->stmt));
1445   }
1446   out->stmt = NEW_STMT(WhileStatement, cond->exp, stmt->stmt);
1447 }
1448
1449 void Parser::onDo(Token &out, Token &stmt, Token &cond) {
1450   out->stmt = NEW_STMT(DoStatement, stmt->stmt, cond->exp);
1451 }
1452
1453 void Parser::onFor(Token &out, Token &expr1, Token &expr2, Token &expr3,
1454                    Token &stmt) {
1455   if (stmt->stmt && stmt->stmt->is(Statement::KindOfStatementList)) {
1456     stmt->stmt = NEW_STMT(BlockStatement,
1457                           dynamic_pointer_cast<StatementList>(stmt->stmt));
1458   }
1459   out->stmt = NEW_STMT(ForStatement, expr1->exp, expr2->exp, expr3->exp,
1460                        stmt->stmt);
1461 }
1462
1463 void Parser::onSwitch(Token &out, Token &expr, Token &cases) {
1464   out->stmt = NEW_STMT(SwitchStatement, expr->exp,
1465                        dynamic_pointer_cast<StatementList>(cases->stmt));
1466 }
1467
1468 void Parser::onCase(Token &out, Token &cases, Token *cond, Token &stmt) {
1469   if (!cases->stmt) {
1470     out->stmt = NEW_STMT0(StatementList);
1471   } else {
1472     out->stmt = cases->stmt;
1473   }
1474   out->stmt->addElement(NEW_STMT(CaseStatement,
1475                                  cond ? cond->exp : ExpressionPtr(),
1476                                  stmt->stmt));
1477 }
1478
1479 void Parser::onBreakContinue(Token &out, bool isBreak, Token* expr) {
1480   uint64_t depth = 1;
1481
1482   if (expr) {
1483     Variant v;
1484     if (!expr->exp->getScalarValue(v)) {
1485       PARSE_ERROR("'%s' operator with non-constant operand is no longer "
1486                   "supported", (isBreak ? "break" : "continue"));
1487     }
1488     if (!v.isInteger() || v.toInt64() <= 0) {
1489       PARSE_ERROR("'%s' operator accepts only positive numbers",
1490                   (isBreak ? "break" : "continue"));
1491     }
1492     depth = static_cast<uint64_t>(v.toInt64());
1493   }
1494
1495   if (isBreak) {
1496     out->stmt = NEW_STMT(BreakStatement, depth);
1497   } else {
1498     out->stmt = NEW_STMT(ContinueStatement, depth);
1499   }
1500 }
1501
1502 void Parser::onReturn(Token &out, Token *expr) {
1503   out->stmt = NEW_STMT(ReturnStatement, expr ? expr->exp : ExpressionPtr());
1504   // When HipHopSyntax is enabled, "yield break" is the only supported method
1505   // for early termination of a generator.
1506   if (!m_funcContexts.empty() &&
1507       (expr || (Scanner::AllowHipHopSyntax & Option::GetScannerType()))) {
1508     FunctionContext& fc = m_funcContexts.back();
1509     if (fc.isGenerator) {
1510       Compiler::Error(InvalidYield, out->stmt);
1511       PARSE_ERROR((Scanner::AllowHipHopSyntax & Option::GetScannerType()) ?
1512         "Cannot mix 'return' and 'yield' in the same function" :
1513         "Generators cannot return values using \"return\"");
1514       return;
1515     }
1516     fc.hasReturn = true;
1517   }
1518 }
1519
1520 void Parser::invalidYield() {
1521   ExpressionPtr exp(new SimpleFunctionCall(BlockScopePtr(),
1522                                            getLocation(),
1523                                            "yield",
1524                                            false,
1525                                            ExpressionListPtr(),
1526                                            ExpressionPtr()));
1527   Compiler::Error(Compiler::InvalidYield, exp);
1528 }
1529
1530 bool Parser::canBeAsyncOrGenerator(string funcName, string clsName) {
1531   if (clsName.empty()) {
1532     return true;
1533   }
1534   if (strcasecmp(funcName.c_str(), clsName.c_str()) == 0) {
1535     return false;
1536   }
1537   if (strncmp(funcName.c_str(), "__", 2) == 0) {
1538     const char *fname = funcName.c_str() + 2;
1539     if (!strcasecmp(fname, "construct") ||
1540         !strcasecmp(fname, "destruct") ||
1541         !strcasecmp(fname, "get") ||
1542         !strcasecmp(fname, "set") ||
1543         !strcasecmp(fname, "isset") ||
1544         !strcasecmp(fname, "unset") ||
1545         !strcasecmp(fname, "call") ||
1546         !strcasecmp(fname, "callstatic") ||
1547         !strcasecmp(fname, "invoke")) {
1548       return false;
1549     }
1550   }
1551   return true;
1552 }
1553
1554 bool Parser::setIsGenerator() {
1555   if (m_funcContexts.empty()) {
1556     invalidYield();
1557     PARSE_ERROR("Yield can only be used inside a function");
1558     return false;
1559   }
1560
1561   FunctionContext& fc = m_funcContexts.back();
1562   if (fc.hasReturn) {
1563     invalidYield();
1564     PARSE_ERROR((Scanner::AllowHipHopSyntax & Option::GetScannerType()) ?
1565       "Cannot mix 'return' and 'yield' in the same function" :
1566       "Generators cannot return values using \"return\"");
1567     return false;
1568   }
1569   if (fc.isAsync) {
1570     invalidYield();
1571     PARSE_ERROR("'yield' is not allowed in async functions.");
1572     return false;
1573   }
1574   fc.isGenerator = true;
1575
1576   if (!canBeAsyncOrGenerator(m_funcName, m_clsName)) {
1577     invalidYield();
1578     PARSE_ERROR("'yield' is not allowed in constructor, destructor, or "
1579                 "magic methods");
1580     return false;
1581   }
1582
1583   return true;
1584 }
1585
1586 void Parser::onYield(Token &out, Token &expr) {
1587   if (setIsGenerator()) {
1588     out->exp = NEW_EXP(YieldExpression, ExpressionPtr(), expr->exp);
1589   }
1590 }
1591
1592 void Parser::onYieldPair(Token &out, Token &key, Token &val) {
1593   if (setIsGenerator()) {
1594     out->exp = NEW_EXP(YieldExpression, key->exp, val->exp);
1595   }
1596 }
1597
1598 void Parser::onYieldBreak(Token &out) {
1599   if (setIsGenerator()) {
1600     out->stmt = NEW_STMT(ReturnStatement, ExpressionPtr());
1601   }
1602 }
1603
1604 void Parser::invalidAwait() {
1605   ExpressionPtr exp(new SimpleFunctionCall(BlockScopePtr(),
1606                                            getLocation(),
1607                                            "async",
1608                                            false,
1609                                            ExpressionListPtr(),
1610                                            ExpressionPtr()));
1611   Compiler::Error(Compiler::InvalidAwait, exp);
1612 }
1613
1614 bool Parser::setIsAsync() {
1615   if (m_funcContexts.empty()) {
1616     invalidAwait();
1617     PARSE_ERROR("'await' can only be used inside a function");
1618     return false;
1619   }
1620
1621   FunctionContext& fc = m_funcContexts.back();
1622   if (fc.isGenerator) {
1623     invalidAwait();
1624     PARSE_ERROR("'await' is not allowed in generators.");
1625     return false;
1626   }
1627   fc.isAsync = true;
1628
1629   if (!canBeAsyncOrGenerator(m_funcName, m_clsName)) {
1630     invalidAwait();
1631     PARSE_ERROR("'await' is not allowed in constructors, destructors, or "
1632                     "magic methods.");
1633   }
1634
1635   return true;
1636 }
1637
1638
1639 void Parser::onAwait(Token &out, Token &expr) {
1640   if (setIsAsync()) {
1641     out->exp = NEW_EXP(AwaitExpression, expr->exp);
1642   }
1643 }
1644
1645 void Parser::onGlobal(Token &out, Token &expr) {
1646   out->stmt = NEW_STMT(GlobalStatement,
1647                        dynamic_pointer_cast<ExpressionList>(expr->exp));
1648 }
1649
1650 void Parser::onGlobalVar(Token &out, Token *exprs, Token &expr) {
1651   ExpressionPtr expList;
1652   if (exprs && exprs->exp) {
1653     expList = exprs->exp;
1654   } else {
1655     expList = NEW_EXP0(ExpressionList);
1656   }
1657   switch (expr->num()) {
1658   case 0:
1659     expList->addElement(NEW_EXP(SimpleVariable, expr->text()));
1660     break;
1661   case 1:
1662     expList->addElement(createDynamicVariable(expr->exp));
1663     break;
1664   default:
1665     assert(false);
1666   }
1667   out->exp = expList;
1668 }
1669
1670 void Parser::onStatic(Token &out, Token &expr) {
1671   out->stmt = NEW_STMT(StaticStatement,
1672                        dynamic_pointer_cast<ExpressionList>(expr->exp));
1673 }
1674
1675 void Parser::onEcho(Token &out, Token &expr, bool html) {
1676   if (html) {
1677     LocationPtr loc = getLocation();
1678     if (loc->line1 == 2 && loc->char1 == 0 && expr->text()[0] == '#') {
1679       // skipping linux interpreter declaration
1680       out->stmt = NEW_STMT0(StatementList);
1681     } else {
1682       ExpressionPtr exp = NEW_EXP(ScalarExpression, T_STRING, expr->text(),
1683                                   true);
1684       ExpressionListPtr expList = NEW_EXP(ExpressionList);
1685       expList->addElement(exp);
1686       out->stmt = NEW_STMT(EchoStatement, expList);
1687     }
1688   } else {
1689     out->stmt = NEW_STMT(EchoStatement,
1690                          dynamic_pointer_cast<ExpressionList>(expr->exp));
1691   }
1692 }
1693
1694 void Parser::onUnset(Token &out, Token &expr) {
1695   out->stmt = NEW_STMT(UnsetStatement,
1696                        dynamic_pointer_cast<ExpressionList>(expr->exp));
1697   m_file->setAttribute(FileScope::ContainsUnset);
1698 }
1699
1700 void Parser::onExpStatement(Token &out, Token &expr) {
1701   ExpStatementPtr exp(NEW_STMT(ExpStatement, expr->exp));
1702   out->stmt = exp;
1703   exp->onParse(m_ar, m_file);
1704 }
1705
1706 void Parser::onForEach(Token &out, Token &arr, Token &name, Token &value,
1707                        Token &stmt) {
1708   if (dynamic_pointer_cast<FunctionCall>(name->exp) ||
1709       dynamic_pointer_cast<FunctionCall>(value->exp)) {
1710     PARSE_ERROR("Can't use return value in write context");
1711   }
1712   if (value->exp && name->num()) {
1713     PARSE_ERROR("Key element cannot be a reference");
1714     return;
1715   }
1716   checkAssignThis(name);
1717   checkAssignThis(value);
1718   if (stmt->stmt && stmt->stmt->is(Statement::KindOfStatementList)) {
1719     stmt->stmt = NEW_STMT(BlockStatement,
1720                           dynamic_pointer_cast<StatementList>(stmt->stmt));
1721   }
1722   out->stmt = NEW_STMT(ForEachStatement, arr->exp, name->exp, name->num() == 1,
1723                        value->exp, value->num() == 1, stmt->stmt);
1724 }
1725
1726 void Parser::onTry(Token &out, Token &tryStmt, Token &className, Token &var,
1727                    Token &catchStmt, Token &catches, Token &finallyStmt) {
1728   StatementPtr stmtList;
1729   if (catches->stmt) {
1730     stmtList = catches->stmt;
1731   } else {
1732     stmtList = NEW_STMT0(StatementList);
1733   }
1734   stmtList->insertElement(NEW_STMT(CatchStatement, className->text(),
1735                                    var->text(), catchStmt->stmt));
1736   out->stmt = NEW_STMT(TryStatement, tryStmt->stmt,
1737                        dynamic_pointer_cast<StatementList>(stmtList),
1738                        finallyStmt->stmt);
1739   if (tryStmt->stmt) {
1740     out->stmt->setLabelScope(stmtList->getLabelScope());
1741   }
1742 }
1743
1744 void Parser::onTry(Token &out, Token &tryStmt, Token &finallyStmt) {
1745   out->stmt = NEW_STMT(TryStatement, tryStmt->stmt,
1746                        dynamic_pointer_cast<StatementList>(NEW_STMT0(StatementList)),
1747                        finallyStmt->stmt);
1748   if (tryStmt->stmt) {
1749     out->stmt->setLabelScope(tryStmt->stmt->getLabelScope());
1750   }
1751 }
1752
1753 void Parser::onCatch(Token &out, Token &catches, Token &className, Token &var,
1754                      Token &stmt) {
1755   StatementPtr stmtList;
1756   if (catches->stmt) {
1757     stmtList = catches->stmt;
1758   } else {
1759     stmtList = NEW_STMT0(StatementList);
1760   }
1761   stmtList->addElement(NEW_STMT(CatchStatement, className->text(),
1762                                 var->text(), stmt->stmt));
1763   out->stmt = stmtList;
1764 }
1765
1766 void Parser::onFinally(Token &out, Token &stmt) {
1767   out->stmt = NEW_STMT(FinallyStatement, stmt->stmt);
1768   // TODO (#3271396) This can be greatly improved. In particular
1769   // even when a finally block exists inside a function it is often
1770   // the case that the unnamed locals state & ret are not needed.
1771   // See task description for further details.
1772   m_file->setAttribute(FileScope::NeedsFinallyLocals);
1773 }
1774
1775 void Parser::onThrow(Token &out, Token &expr) {
1776   out->stmt = NEW_STMT(ThrowStatement, expr->exp);
1777 }
1778
1779 void Parser::onClosureStart(Token &name) {
1780   if (!m_funcName.empty()) {
1781     m_containingFuncName = m_funcName;
1782   } else {
1783     // pseudoMain
1784   }
1785   onFunctionStart(name, true);
1786 }
1787
1788 Token Parser::onClosure(ClosureType type,
1789                         Token* modifiers,
1790                         Token& ref,
1791                         Token& params,
1792                         Token& cparams,
1793                         Token& stmts) {
1794   Token out;
1795   Token name;
1796
1797   Token retIgnore;
1798   auto stmt = onFunctionHelper(
1799     FunctionType::Closure,
1800     modifiers,
1801     retIgnore,
1802     ref,
1803     nullptr,
1804     params,
1805     stmts,
1806     nullptr,
1807     true
1808   );
1809
1810   ExpressionListPtr vars = dynamic_pointer_cast<ExpressionList>(cparams->exp);
1811   if (vars) {
1812     for (int i = vars->getCount() - 1; i >= 0; i--) {
1813       ParameterExpressionPtr param(
1814         dynamic_pointer_cast<ParameterExpression>((*vars)[i]));
1815       if (param->getName() == "this") {
1816         PARSE_ERROR("Cannot use $this as lexical variable");
1817       }
1818     }
1819   }
1820
1821   ClosureExpressionPtr closure = NEW_EXP(
1822     ClosureExpression,
1823     type,
1824     dynamic_pointer_cast<FunctionStatement>(stmt),
1825     vars
1826   );
1827   closure->getClosureFunction()->setContainingClosure(closure);
1828   out->exp = closure;
1829
1830   return out;
1831 }
1832
1833 Token Parser::onExprForLambda(const Token& expr) {
1834   auto stmt = NEW_STMT(ReturnStatement, expr.exp);
1835   Token ret;
1836   ret.stmt = NEW_STMT0(StatementList);
1837   ret.stmt->addElement(stmt);
1838   return ret;
1839 }
1840
1841 void Parser::onClosureParam(Token &out, Token *params, Token &param,
1842                             bool ref) {
1843   ExpressionPtr expList;
1844   if (params) {
1845     expList = params->exp;
1846   } else {
1847     expList = NEW_EXP0(ExpressionList);
1848   }
1849   expList->addElement(NEW_EXP(ParameterExpression, TypeAnnotationPtr(),
1850                               m_scanner.isHHSyntaxEnabled(), param->text(),
1851                               ref, 0, ExpressionPtr(), ExpressionPtr()));
1852   out->exp = expList;
1853 }
1854
1855 void Parser::onLabel(Token &out, Token &label) {
1856   out->stmt = NEW_STMT(LabelStatement, label.text());
1857 }
1858
1859 void Parser::onGoto(Token &out, Token &label, bool limited) {
1860   out->stmt = NEW_STMT(GotoStatement, label.text());
1861 }
1862
1863 void Parser::onTypedef(Token& out, const Token& name, const Token& type) {
1864   // Note: we don't always get TypeAnnotations (e.g. for shape types
1865   // currently).
1866   auto const annot = type.typeAnnotation
1867     ? type.typeAnnotation
1868     : std::make_shared<TypeAnnotation>(type.text(), TypeAnnotationPtr());
1869
1870   auto td_stmt = NEW_STMT(TypedefStatement, name.text(), annot);
1871   td_stmt->onParse(m_ar, m_file);
1872   out->stmt = td_stmt;
1873 }
1874
1875 void Parser::onTypeAnnotation(Token& out, const Token& name,
1876                                           const Token& typeArgs) {
1877   out.set(name.num(), name.text());
1878   out.typeAnnotation = TypeAnnotationPtr(
1879     new TypeAnnotation(name.text(), typeArgs.typeAnnotation));
1880   if (isTypeVar(name.text())) {
1881     out.typeAnnotation->setTypeVar();
1882   }
1883 }
1884
1885 void Parser::onTypeList(Token& type1, const Token& type2) {
1886   if (!type1.typeAnnotation) {
1887     PARSE_ERROR("Missing type in type list");
1888   }
1889   if (type2.num() != 0 && !type1.typeAnnotation) {
1890     PARSE_ERROR("Missing type in type list");
1891   }
1892   if (type2.typeAnnotation) {
1893     type1.typeAnnotation->appendToTypeList(type2.typeAnnotation);
1894   }
1895 }
1896
1897 void Parser::onTypeSpecialization(Token& type, char specialization) {
1898   if (type.typeAnnotation) {
1899     switch (specialization) {
1900     case '?':
1901       type.typeAnnotation->setNullable();
1902       break;
1903     case '@':
1904       type.typeAnnotation->setSoft();
1905       break;
1906     case 't':
1907       type.typeAnnotation->setTuple();
1908       break;
1909     case 'f':
1910       type.typeAnnotation->setFunction();
1911       break;
1912     case 'x':
1913       type.typeAnnotation->setXHP();
1914       break;
1915     }
1916   }
1917 }
1918
1919 void Parser::onQuery(Token &out, Token &head, Token &body) {
1920   auto qe = NEW_EXP(QueryExpression, head.exp, body.exp);
1921   qe->doRewrites(m_ar, m_file);
1922   out->exp = qe;
1923 }
1924
1925 void appendList(ExpressionListPtr expList, Token *exps) {
1926   if (exps != nullptr) {
1927     assert(exps->exp->is(Expression::KindOfExpressionList));
1928     ExpressionListPtr el(static_pointer_cast<ExpressionList>(exps->exp));
1929     for (unsigned int i = 0; i < el->getCount(); i++) {
1930       if ((*el)[i]) expList->addElement((*el)[i]);
1931     }
1932   }
1933 }
1934
1935 void Parser::onQueryBody(
1936   Token &out, Token *clauses, Token &select, Token *cont) {
1937   ExpressionListPtr expList(NEW_EXP0(ExpressionList));
1938   appendList(expList, clauses);
1939   expList->addElement(select.exp);
1940   if (cont != nullptr) expList->addElement(cont->exp);
1941   out->exp = expList;
1942 }
1943
1944 void Parser::onQueryBodyClause(Token &out, Token *clauses, Token &clause) {
1945   ExpressionPtr expList;
1946   if (clauses && clauses->exp) {
1947     expList = clauses->exp;
1948   } else {
1949     expList = NEW_EXP0(ExpressionList);
1950   }
1951   expList->addElement(clause->exp);
1952   out->exp = expList;
1953 }
1954
1955 void Parser::onFromClause(Token &out, Token &var, Token &coll) {
1956   out->exp = NEW_EXP(FromClause, var.text(), coll.exp);
1957 }
1958
1959 void Parser::onLetClause(Token &out, Token &var, Token &expr) {
1960   out->exp = NEW_EXP(LetClause, var.text(), expr.exp);
1961 }
1962
1963 void Parser::onWhereClause(Token &out, Token &expr) {
1964   out->exp = NEW_EXP(WhereClause, expr.exp);
1965 }
1966
1967 void Parser::onJoinClause(Token &out, Token &var, Token &coll,
1968   Token &left, Token &right) {
1969   out->exp = NEW_EXP(JoinClause, var.text(), coll.exp,
1970                      left.exp, right.exp, "");
1971 }
1972
1973 void Parser::onJoinIntoClause(Token &out, Token &var, Token &coll,
1974   Token &left, Token &right, Token &group) {
1975   out->exp = NEW_EXP(JoinClause, var.text(), coll.exp,
1976                      left.exp, right.exp, group.text());
1977 }
1978
1979 void Parser::onOrderbyClause(Token &out, Token &orderings) {
1980   out->exp = NEW_EXP(OrderbyClause, orderings.exp);
1981 }
1982
1983 void Parser::onOrdering(Token &out, Token *orderings, Token &ordering) {
1984   ExpressionPtr expList;
1985   if (orderings && orderings->exp) {
1986     expList = orderings->exp;
1987   } else {
1988     expList = NEW_EXP0(ExpressionList);
1989   }
1990   expList->addElement(ordering->exp);
1991   out->exp = expList;
1992 }
1993
1994 void Parser::onOrderingExpr(Token &out, Token &expr, Token *direction) {
1995   out->exp = NEW_EXP(Ordering, expr.exp, (direction) ? direction->text() : "");
1996 }
1997
1998 void Parser::onSelectClause(Token &out, Token &expr) {
1999   out->exp = NEW_EXP(SelectClause, expr.exp);
2000 }
2001
2002 void Parser::onGroupClause(Token &out, Token &coll, Token &key) {
2003   out->exp = NEW_EXP(GroupClause, coll.exp, key.exp);
2004 }
2005
2006 void Parser::onIntoClause(Token &out, Token &var, Token &query) {
2007   out->exp = NEW_EXP(IntoClause, var.text(), query.exp);
2008 }
2009
2010 ///////////////////////////////////////////////////////////////////////////////
2011 // namespace support
2012
2013 //////////////////// AliasTable /////////////////////
2014
2015 Parser::AliasTable::AliasTable(const hphp_string_imap<std::string>& autoAliases,
2016                                std::function<bool ()> autoOracle)
2017   : m_autoAliases(autoAliases), m_autoOracle(autoOracle) {
2018   if (!m_autoOracle) {
2019     setFalseOracle();
2020   }
2021 }
2022
2023 void Parser::AliasTable::setFalseOracle() {
2024   m_autoOracle = [] () { return false; };
2025 }
2026
2027 std::string Parser::AliasTable::getName(std::string alias) {
2028   auto it = m_aliases.find(alias);
2029   if (it != m_aliases.end()) {
2030     return it->second.name;
2031   }
2032   auto autoIt = m_autoAliases.find(alias);
2033   if (autoIt != m_autoAliases.end()) {
2034     set(alias, autoIt->second, AliasTable::AliasType::USE);
2035     return autoIt->second;
2036   }
2037   return "";
2038 }
2039
2040 std::string Parser::AliasTable::getDefName(std::string alias) {
2041   auto it = m_aliases.find(alias);
2042   if (it != m_aliases.end() && it->second.type == AliasType::DEF) {
2043     return it->second.name;
2044   }
2045   return "";
2046 }
2047
2048 std::string Parser::AliasTable::getUseName(std::string alias) {
2049   auto it = m_aliases.find(alias);
2050   if (it != m_aliases.end() && it->second.type == AliasType::USE) {
2051     return it->second.name;
2052   }
2053   return "";
2054 }
2055
2056 bool Parser::AliasTable::isAliased(std::string alias) {
2057   if (isUseType(alias)) {
2058     return true;
2059   }
2060   return m_autoOracle() && m_autoAliases.find(alias) != m_autoAliases.end();
2061 }
2062
2063 bool Parser::AliasTable::isAutoType(std::string alias) {
2064   return m_autoOracle() && m_autoAliases.find(alias) != m_autoAliases.end();
2065 }
2066
2067 bool Parser::AliasTable::isUseType(std::string alias) {
2068   auto it = m_aliases.find(alias);
2069   return it != m_aliases.end() && it->second.type == AliasType::USE;
2070 }
2071
2072 bool Parser::AliasTable::isDefType(std::string alias) {
2073   auto it = m_aliases.find(alias);
2074   return it != m_aliases.end() && it->second.type == AliasType::DEF;
2075 }
2076
2077 void Parser::AliasTable::set(std::string alias,
2078                              std::string name,
2079                              AliasType type) {
2080   m_aliases[alias] = (NameEntry){name, type};
2081 }
2082
2083 /*
2084  * To be called when we enter a fresh namespace.
2085  */
2086 void Parser::AliasTable::clear() {
2087   m_aliases.clear();
2088 }
2089
2090 //////////////////////////////////////////////////////
2091
2092
2093 /*
2094  * We auto-alias classes only on HH mode.
2095  */
2096 bool Parser::isAutoAliasOn() {
2097   return m_scanner.isHHSyntaxEnabled();
2098 }
2099
2100 hphp_string_imap<std::string> Parser::getAutoAliasedClassesHelper() {
2101   hphp_string_imap<std::string> autoAliases;
2102   typedef AliasTable::AliasEntry AliasEntry;
2103   std::vector<AliasEntry> aliases {
2104     (AliasEntry){"Traversable", "HH\\Traversable"},
2105     (AliasEntry){"KeyedTraversable", "HH\\KeyedTraversable"},
2106     (AliasEntry){"Iterator", "HH\\Iterator"},
2107     (AliasEntry){"KeyedIterator", "HH\\KeyedIterator"},
2108     (AliasEntry){"Iterable", "HH\\Iterable"},
2109     (AliasEntry){"KeyedIterable", "HH\\KeyedIterable"},
2110     (AliasEntry){"Collection", "HH\\Collection"},
2111     (AliasEntry){"Vector", "HH\\Vector"},
2112     (AliasEntry){"Map", "HH\\Map"},
2113     (AliasEntry){"StableMap", "HH\\Map"}, // Merging with Map
2114     (AliasEntry){"Set", "HH\\Set"},
2115     (AliasEntry){"Pair", "HH\\Pair"},
2116     (AliasEntry){"ImmVector", "HH\\ImmVector"},
2117     (AliasEntry){"ImmMap", "HH\\ImmMap"},
2118     (AliasEntry){"ImmSet", "HH\\ImmSet"},
2119     (AliasEntry){"InvariantException", "HH\\InvariantException"},
2120
2121     (AliasEntry){"bool", "HH\\bool"},
2122     (AliasEntry){"boolean", "HH\\bool"},
2123     (AliasEntry){"int", "HH\\int"},
2124     (AliasEntry){"integer", "HH\\int"},
2125     (AliasEntry){"float", "HH\\float"},
2126     (AliasEntry){"double", "HH\\float"},
2127     (AliasEntry){"real", "HH\\float"},
2128     (AliasEntry){"num", "HH\\num"},
2129     (AliasEntry){"string", "HH\\string"},
2130     (AliasEntry){"resource", "HH\\resource"},
2131     (AliasEntry){"mixed", "HH\\mixed"},
2132   };
2133   for (auto entry : aliases) {
2134     autoAliases[entry.alias] = entry.name;
2135   }
2136   return autoAliases;
2137 }
2138
2139 const hphp_string_imap<std::string>& Parser::getAutoAliasedClasses() {
2140   static auto autoAliases = getAutoAliasedClassesHelper();
2141   return autoAliases;
2142 }
2143
2144 void Parser::nns(int token, const std::string& text) {
2145   if (m_nsState == SeenNamespaceStatement && token != ';') {
2146     error("No code may exist outside of namespace {}: %s",
2147           getMessage().c_str());
2148     return;
2149   }
2150
2151   if (m_nsState == SeenNothing && !text.empty() && token != T_DECLARE &&
2152       token != ';') {
2153     m_nsState = SeenNonNamespaceStatement;
2154   }
2155 }
2156
2157 void Parser::onNamespaceStart(const std::string &ns,
2158                               bool file_scope /* =false */) {
2159   if (m_nsState == SeenNonNamespaceStatement) {
2160     error("Namespace declaration statement has to be the very first "
2161           "statement in the script: %s", getMessage().c_str());
2162     return;
2163   }
2164   if (m_nsState != SeenNothing && file_scope != m_nsFileScope) {
2165     error("Cannot mix bracketed namespace declarations with unbracketed "
2166           "namespace declarations");
2167   }
2168
2169   m_nsState = InsideNamespace;
2170   m_nsFileScope = file_scope;
2171   pushComment();
2172   m_namespace = ns;
2173   m_aliasTable.clear();
2174 }
2175
2176 void Parser::onNamespaceEnd() {
2177   m_nsState = SeenNamespaceStatement;
2178 }
2179
2180 void Parser::onUse(const std::string &ns, const std::string &as) {
2181   string key = as;
2182   if (key.empty()) {
2183     size_t pos = ns.rfind(NAMESPACE_SEP);
2184     if (pos == string::npos) {
2185       key = ns;
2186     } else {
2187       key = ns.substr(pos + 1);
2188     }
2189   }
2190
2191   // It's not an error if the alias already exists but is auto-imported.
2192   // In that case, it gets replaced. It prompts an error if it is not
2193   // auto-imported and 'use' statement is trying to replace it.
2194   if (m_aliasTable.isUseType(key)) {
2195     error("Cannot use %s as %s because the name is already in use: %s",
2196           ns.c_str(), key.c_str(), getMessage().c_str());
2197     return;
2198   }
2199   if (m_aliasTable.isDefType(key)) {
2200     auto defName = m_aliasTable.getDefName(key);
2201     if (strcasecmp(defName.c_str(), ns.c_str())) {
2202       error("Cannot use %s as %s because the name is already in use: %s",
2203             ns.c_str(), key.c_str(), getMessage().c_str());
2204       return;
2205     }
2206   }
2207
2208   m_aliasTable.set(key, ns, AliasTable::AliasType::USE);
2209 }
2210
2211 std::string Parser::nsDecl(const std::string &name) {
2212   if (m_namespace.empty()) {
2213     return name;
2214   }
2215   return m_namespace + NAMESPACE_SEP + name;
2216 }
2217
2218 std::string Parser::resolve(const std::string &ns, bool cls) {
2219   size_t pos = ns.find(NAMESPACE_SEP);
2220   string alias = (pos != string::npos) ? ns.substr(0, pos) : ns;
2221
2222   if (m_aliasTable.isAliased(alias)) {
2223     auto name = m_aliasTable.getName(alias);
2224     // Was it a namespace alias?
2225     if (pos != string::npos) {
2226       return name + ns.substr(pos);
2227     }
2228     // Only classes can appear directly in "use" statements
2229     if (cls) {
2230       return name;
2231     }
2232   }
2233
2234   // Classes don't fallback to the global namespace.
2235   if (cls) {
2236     if (!strcasecmp("self", ns.c_str()) ||
2237         !strcasecmp("parent", ns.c_str())) {
2238       return ns;
2239     }
2240     return nsDecl(ns);
2241   }
2242
2243   // if qualified name, prepend current namespace
2244   if (pos != string::npos) {
2245     return nsDecl(ns);
2246   }
2247
2248   // unqualified name in global namespace
2249   if (m_namespace.empty()) {
2250     return ns;
2251   }
2252
2253   if (!strcasecmp("true", ns.c_str()) ||
2254       !strcasecmp("false", ns.c_str()) ||
2255       !strcasecmp("null", ns.c_str())) {
2256     return ns;
2257   }
2258   return nsDecl(ns);
2259 }
2260
2261 void Parser::invalidateGoto(TStatementPtr stmt, GotoError error) {
2262   GotoStatement *gs = (GotoStatement*) stmt;
2263   assert(gs);
2264   gs->invalidate(error);
2265 }
2266
2267 void Parser::invalidateLabel(TStatementPtr stmt) {
2268   LabelStatement *ls = (LabelStatement*) stmt;
2269   assert(ls);
2270   ls->invalidate();
2271 }
2272
2273 TStatementPtr Parser::extractStatement(ScannerToken *stmt) {
2274   Token *t = (Token*) stmt;
2275   return t->stmt.get();
2276 }
2277
2278 ///////////////////////////////////////////////////////////////////////////////
2279
2280 bool Parser::hasType(Token &type) {
2281   if (!type.text().empty()) {
2282     if (!m_scanner.isHHSyntaxEnabled()) {
2283       PARSE_ERROR("Type hint is not enabled");
2284       return false;
2285     }
2286     return true;
2287   }
2288   return false;
2289 }
2290
2291 void Parser::registerAlias(std::string name) {
2292   size_t pos = name.rfind(NAMESPACE_SEP);
2293   if (pos != string::npos) {
2294     string key = name.substr(pos + 1);
2295     if (m_aliasTable.isUseType(key)) {
2296       auto useName = m_aliasTable.getUseName(key);
2297       if (strcasecmp(useName.c_str(), name.c_str())) {
2298         error("Cannot declare class %s because the name is already in use: %s",
2299               name.c_str(), getMessage().c_str());
2300         return;
2301       }
2302     } else {
2303       m_aliasTable.set(key, name, AliasTable::AliasType::DEF);
2304     }
2305   }
2306 }
2307
2308 ///////////////////////////////////////////////////////////////////////////////
2309 }}