hphp/runtime/ext/ext_misc.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-2014 Facebook, Inc. (http://www.facebook.com)     |
   6    | Copyright (c) 1997-2010 The PHP Group                                |
   7    +----------------------------------------------------------------------+
   8    | This source file is subject to version 3.01 of the PHP license,      |
   9    | that is bundled with this package in the file LICENSE, and is        |
  10    | available through the world-wide-web at the following url:           |
  11    | http://www.php.net/license/3_01.txt                                  |
  12    | If you did not receive a copy of the PHP license and are unable to   |
  13    | obtain it through the world-wide-web, please send a note to          |
  14    | license@php.net so we can mail you a copy immediately.               |
  15    +----------------------------------------------------------------------+
  16 */
  17
  18 #include "hphp/runtime/ext/ext_misc.h"
  19 #include <limits>
  20
  21 #include "hphp/runtime/server/server-stats.h"
  22 #include "hphp/runtime/base/exceptions.h"
  23 #include "hphp/runtime/base/zend-pack.h"
  24 #include "hphp/runtime/base/hphp-system.h"
  25 #include "hphp/runtime/base/runtime-option.h"
  26 #include "hphp/runtime/base/strings.h"
  27 #include "hphp/runtime/ext/ext_math.h"
  28 #include "hphp/runtime/vm/bytecode.h"
  29 #include "hphp/runtime/vm/type-profile.h"
  30 #include "hphp/parser/scanner.h"
  31 #include "hphp/runtime/base/class-info.h"
  32 #include "hphp/runtime/vm/jit/mc-generator.h"
  33 #include "hphp/runtime/vm/jit/timer.h"
  34 #include "hphp/runtime/vm/jit/translator.h"
  35 #include "hphp/runtime/vm/jit/translator-inline.h"
  36 #include "hphp/system/constants.h"
  37 #include "hphp/util/logger.h"
  38
  39 namespace HPHP {
  40
  41 IMPLEMENT_THREAD_LOCAL(std::string, s_misc_highlight_default_string);
  42 IMPLEMENT_THREAD_LOCAL(std::string, s_misc_highlight_default_comment);
  43 IMPLEMENT_THREAD_LOCAL(std::string, s_misc_highlight_default_keyword);
  44 IMPLEMENT_THREAD_LOCAL(std::string, s_misc_highlight_default_default);
  45 IMPLEMENT_THREAD_LOCAL(std::string, s_misc_highlight_default_html);
  46 IMPLEMENT_THREAD_LOCAL(std::string, s_misc_display_errors);
  47
  48 const std::string s_1("1"), s_2("2"), s_stdout("stdout"), s_stderr("stderr");
  49
  50 static String HHVM_FUNCTION(server_warmup_status) {
  51   // Fail if we jitted more than 25kb of code.
  52   size_t begin, end;
  53   JIT::mcg->codeEmittedThisRequest(begin, end);
  54   auto const diff = end - begin;
  55   auto constexpr kMaxTCBytes = 25 << 10;
  56   if (diff > kMaxTCBytes) {
  57     return folly::format("Translation cache grew by {} bytes to {} bytes.",
  58                          diff, begin).str();
  59   }
  60
  61   // Fail if we spent more than 0.5ms in the JIT.
  62   auto const jittime = JIT::Timer::CounterValue(JIT::Timer::translate);
  63   auto constexpr kMaxJitTimeNS = 500000;
  64   if (jittime.total > kMaxJitTimeNS) {
  65     return folly::format("Spent {}us in the JIT.", jittime.total / 1000).str();
  66   }
  67
  68   if (shouldProfile()) {
  69     return "Warmup profiling is still enabled.";
  70   }
  71
  72   if (requestCount() <= RuntimeOption::EvalJitProfileRequests) {
  73     return "PGO profiling translations are still enabled.";
  74   }
  75
  76   return empty_string;
  77 }
  78
  79 static class MiscExtension : public Extension {
  80 public:
  81   MiscExtension() : Extension("misc", k_PHP_VERSION.c_str()) { }
  82   void threadInit() {
  83     IniSetting::Bind(
  84       this, IniSetting::PHP_INI_ALL,
  85       "highlight.string", "#DD0000",
  86       s_misc_highlight_default_string.get()
  87     );
  88     IniSetting::Bind(
  89       this, IniSetting::PHP_INI_ALL,
  90       "highlight.comment", "#FF8000",
  91       s_misc_highlight_default_comment.get()
  92     );
  93     IniSetting::Bind(
  94       this, IniSetting::PHP_INI_ALL,
  95       "highlight.keyword", "#007700",
  96       s_misc_highlight_default_keyword.get()
  97     );
  98     IniSetting::Bind(
  99       this, IniSetting::PHP_INI_ALL,
 100       "highlight.default", "#0000BB",
 101       s_misc_highlight_default_default.get()
 102     );
 103     IniSetting::Bind(
 104       this, IniSetting::PHP_INI_ALL,
 105       "highlight.html", "#000000",
 106       s_misc_highlight_default_html.get()
 107     );
 108     IniSetting::Bind(
 109       this, IniSetting::PHP_INI_ALL,
 110       "display_errors", RuntimeOption::EnableHipHopSyntax ? "stderr" : "1",
 111       IniSetting::SetAndGet<std::string>(
 112         [](const std::string& value) {
 113           if (value == s_1 || value == s_stdout) {
 114             Logger::SetStandardOut(stdout);
 115             return true;
 116           }
 117           if (value == s_2 || value == s_stderr) {
 118             Logger::SetStandardOut(stderr);
 119             return true;
 120           }
 121           return false;
 122         },
 123         nullptr
 124       ),
 125       s_misc_display_errors.get()
 126     );
 127   }
 128
 129   virtual void moduleInit() override {
 130     HHVM_FALIAS(HH\\server_warmup_status, server_warmup_status);
 131     loadSystemlib();
 132   }
 133 } s_misc_extension;
 134
 135 // Make sure "tokenizer" gets added to the list of extensions
 136 IMPLEMENT_DEFAULT_EXTENSION_VERSION(tokenizer, 0.1);
 137
 138 const double k_INF = std::numeric_limits<double>::infinity();
 139 const double k_NAN = std::numeric_limits<double>::quiet_NaN();
 140 const bool k_PHP_DEBUG =
 141 #if DEBUG
 142         true;
 143 #else
 144         false;
 145 #endif
 146
 147 const int64_t k_PHP_MAXPATHLEN = MAXPATHLEN;
 148
 149 ///////////////////////////////////////////////////////////////////////////////
 150
 151 int64_t f_connection_aborted() {
 152   return f_connection_status() == k_CONNECTION_ABORTED;
 153 }
 154
 155 int64_t f_connection_status() {
 156   return k_CONNECTION_NORMAL;
 157 }
 158
 159 int64_t f_connection_timeout() {
 160   return f_connection_status() == k_CONNECTION_TIMEOUT;
 161 }
 162
 163 static Class* getClassByName(const char* name, int len) {
 164   Class* cls = nullptr;
 165   // translate "self" or "parent"
 166   if (len == 4 && !memcmp(name, "self", 4)) {
 167     cls = g_context->getContextClass();
 168     if (!cls) {
 169       throw FatalErrorException("Cannot access self:: "
 170                                 "when no class scope is active");
 171     }
 172   } else if (len == 6 && !memcmp(name, "parent", 6)) {
 173     cls = g_context->getParentContextClass();
 174     if (!cls) {
 175       throw FatalErrorException("Cannot access parent");
 176     }
 177   } else if (len == 6 && !memcmp(name, "static", 6)) {
 178     CallerFrame cf;
 179     auto ar = cf();
 180     if (ar) {
 181       if (ar->hasThis()) {
 182         cls = ar->getThis()->getVMClass();
 183       } else if (ar->hasClass()) {
 184         cls = ar->getClass();
 185       }
 186     }
 187     if (!cls) {
 188       throw FatalErrorException("Cannot access static:: "
 189                                 "when no class scope is active");
 190     }
 191   } else {
 192     String className(name, len, CopyString);
 193     cls = Unit::loadClass(className.get());
 194   }
 195   return cls;
 196 }
 197
 198 Variant f_constant(const String& name) {
 199   if (!name.get()) return init_null();
 200   const char *data = name.data();
 201   int len = name.length();
 202
 203   char *colon;
 204   if ((colon = (char*)memchr(data, ':', len)) && colon[1] == ':') {
 205     // class constant
 206     int classNameLen = colon - data;
 207     char *constantName = colon + 2;
 208     Class* cls = getClassByName(data, classNameLen);
 209     if (cls) {
 210       String cnsName(constantName, data + len - constantName, CopyString);
 211       Cell cns = cls->clsCnsGet(cnsName.get());
 212       if (cns.m_type != KindOfUninit) {
 213         return cellAsCVarRef(cns);
 214       }
 215     }
 216     raise_warning("Couldn't find constant %s", data);
 217   } else {
 218     auto const cns = Unit::loadCns(name.get());
 219     if (cns) return tvAsCVarRef(cns);
 220   }
 221
 222   return init_null();
 223 }
 224
 225 bool f_define(const String& name, const Variant& value,
 226               bool case_insensitive /* = false */) {
 227   if (case_insensitive) {
 228     raise_warning(Strings::CONSTANTS_CASE_SENSITIVE);
 229   }
 230   return Unit::defCns(name.get(), value.asCell());
 231 }
 232
 233 bool f_defined(const String& name, bool autoload /* = true */) {
 234   if (!name.get()) return false;
 235   const char *data = name.data();
 236   int len = name.length();
 237
 238   char *colon;
 239   if ((colon = (char*)memchr(data, ':', len)) && colon[1] == ':') {
 240     // class constant
 241     int classNameLen = colon - data;
 242     char *constantName = colon + 2;
 243     Class* cls = getClassByName(data, classNameLen);
 244     if (cls) {
 245       String cnsName(constantName, data + len - constantName, CopyString);
 246       return cls->clsCnsGet(cnsName.get()).m_type != KindOfUninit;
 247     }
 248     return false;
 249   } else {
 250     auto* cb = autoload ? Unit::loadCns : Unit::lookupCns;
 251     return cb(name.get());
 252   }
 253 }
 254
 255 Variant f_die(const Variant& status /* = null_variant */) {
 256   return f_exit(status);
 257 }
 258
 259 Variant f_exit(const Variant& status /* = null_variant */) {
 260   if (status.isString()) {
 261     echo(status.toString());
 262     throw ExitException(0);
 263   }
 264   throw ExitException(status.toInt32());
 265 }
 266
 267 void f___halt_compiler() {
 268   // do nothing
 269 }
 270
 271 int64_t f_ignore_user_abort(bool setting /* = false */) {
 272   return 0;
 273 }
 274
 275 Variant f_pack(int _argc, const String& format, const Array& _argv /* = null_array */) {
 276   return ZendPack().pack(format, _argv);
 277 }
 278
 279 int64_t f_sleep(int seconds) {
 280   IOStatusHelper io("sleep");
 281   Transport *transport = g_context->getTransport();
 282   if (transport) {
 283     transport->incSleepTime(seconds);
 284   }
 285   sleep(seconds);
 286   return 0;
 287 }
 288
 289 void f_usleep(int micro_seconds) {
 290   IOStatusHelper io("usleep");
 291   Transport *transport = g_context->getTransport();
 292   if (transport) {
 293     transport->incuSleepTime(micro_seconds);
 294   }
 295   usleep(micro_seconds);
 296 }
 297
 298 static void recordNanosleepTime(
 299   const struct timespec &req,
 300   const struct timespec *rem
 301 ) {
 302   Transport *transport = g_context->getTransport();
 303   if (transport) {
 304     int64_t req_s = req.tv_sec;
 305     int32_t req_n = req.tv_nsec;
 306     int64_t rem_s = 0;
 307     int32_t rem_n = 0;
 308
 309     if (rem) {
 310       rem_s = rem->tv_sec;
 311       rem_n = rem->tv_nsec;
 312     }
 313
 314     int32_t nanos = req_n - rem_n;
 315     int64_t seconds = req_s - rem_s;
 316     if (nanos < 0) {
 317       nanos += 1000000000;
 318       seconds--;
 319     }
 320
 321     transport->incnSleepTime(seconds, nanos);
 322   }
 323 }
 324
 325 const StaticString
 326   s_seconds("seconds"),
 327   s_nanoseconds("nanoseconds");
 328
 329 Variant f_time_nanosleep(int seconds, int nanoseconds) {
 330   if (seconds < 0) {
 331     throw_invalid_argument("seconds: cannot be negative");
 332     return false;
 333   }
 334   if (nanoseconds < 0 || nanoseconds > 999999999) {
 335     throw_invalid_argument("nanoseconds: has to be 0 to 999999999");
 336     return false;
 337   }
 338
 339   struct timespec req, rem;
 340   req.tv_sec = (time_t)seconds;
 341   req.tv_nsec = nanoseconds;
 342
 343   IOStatusHelper io("nanosleep");
 344   if (!nanosleep(&req, &rem)) {
 345     recordNanosleepTime(req, nullptr);
 346     return true;
 347   }
 348
 349   recordNanosleepTime(req, &rem);
 350   if (errno == EINTR) {
 351     return make_map_array(s_seconds, (int64_t)rem.tv_sec,
 352                           s_nanoseconds, (int64_t)rem.tv_nsec);
 353   }
 354   return false;
 355 }
 356
 357 bool f_time_sleep_until(double timestamp) {
 358   struct timeval tm;
 359   if (gettimeofday((struct timeval *)&tm, NULL) != 0) {
 360     return false;
 361   }
 362
 363   double c_ts = (double)(timestamp - tm.tv_sec - tm.tv_usec / 1000000.0);
 364   if (c_ts < 0) {
 365     throw_invalid_argument
 366       ("timestamp: Sleep until to time is less than current time");
 367     return false;
 368   }
 369
 370   struct timespec req, rem;
 371   req.tv_sec = (time_t)c_ts;
 372   req.tv_nsec = (long)((c_ts - req.tv_sec) * 1000000000.0);
 373
 374   IOStatusHelper io("nanosleep");
 375   while (nanosleep(&req, &rem)) {
 376     recordNanosleepTime(req, &rem);
 377     if (errno != EINTR) return false;
 378     req.tv_sec = rem.tv_sec;
 379     req.tv_nsec = rem.tv_nsec;
 380   }
 381   recordNanosleepTime(req, nullptr);
 382   return true;
 383 }
 384
 385 String f_uniqid(const String& prefix /* = null_string */,
 386                 bool more_entropy /* = false */) {
 387   if (!more_entropy) {
 388     Transport *transport = g_context->getTransport();
 389     if (transport) {
 390       transport->incuSleepTime(1);
 391     }
 392     usleep(1);
 393   }
 394
 395   struct timeval tv;
 396   gettimeofday((struct timeval *)&tv, NULL);
 397   int sec = (int)tv.tv_sec;
 398   int usec = (int)(tv.tv_usec % 0x100000);
 399
 400   String uniqid(prefix.size() + 64, ReserveString);
 401   auto ptr = uniqid.bufferSlice().ptr;
 402   auto capacity = uniqid.get()->capacity();
 403   int64_t len;
 404   if (more_entropy) {
 405     len = snprintf(ptr, capacity, "%s%08x%05x%.8F",
 406                    prefix.c_str(), sec, usec, math_combined_lcg() * 10);
 407   } else {
 408     len = snprintf(ptr, capacity, "%s%08x%05x",
 409                    prefix.c_str(), sec, usec);
 410   }
 411   uniqid.setSize(len);
 412   return uniqid;
 413 }
 414
 415 Variant f_unpack(const String& format, const String& data) {
 416   return ZendPack().unpack(format, data);
 417 }
 418
 419 Array f_sys_getloadavg() {
 420   double load[3];
 421   getloadavg(load, 3);
 422   return make_packed_array(load[0], load[1], load[2]);
 423 }
 424
 425 // We want token IDs to remain stable regardless of how we change the
 426 // internals of the parser. Thus, we maintain a mapping from internal
 427 // token IDs to stable "user token IDs" and only expose the user token
 428 // IDs to the PHP application.
 429
 430 const int UserTokenId_T_REQUIRE_ONCE = 258;
 431 const int UserTokenId_T_REQUIRE = 259;
 432 const int UserTokenId_T_EVAL = 260;
 433 const int UserTokenId_T_INCLUDE_ONCE = 261;
 434 const int UserTokenId_T_INCLUDE = 262;
 435 const int UserTokenId_T_LOGICAL_OR = 263;
 436 const int UserTokenId_T_LOGICAL_XOR = 264;
 437 const int UserTokenId_T_LOGICAL_AND = 265;
 438 const int UserTokenId_T_PRINT = 266;
 439 const int UserTokenId_T_SR_EQUAL = 267;
 440 const int UserTokenId_T_SL_EQUAL = 268;
 441 const int UserTokenId_T_XOR_EQUAL = 269;
 442 const int UserTokenId_T_OR_EQUAL = 270;
 443 const int UserTokenId_T_AND_EQUAL = 271;
 444 const int UserTokenId_T_MOD_EQUAL = 272;
 445 const int UserTokenId_T_CONCAT_EQUAL = 273;
 446 const int UserTokenId_T_DIV_EQUAL = 274;
 447 const int UserTokenId_T_MUL_EQUAL = 275;
 448 const int UserTokenId_T_MINUS_EQUAL = 276;
 449 const int UserTokenId_T_PLUS_EQUAL = 277;
 450 const int UserTokenId_T_BOOLEAN_OR = 278;
 451 const int UserTokenId_T_BOOLEAN_AND = 279;
 452 const int UserTokenId_T_IS_NOT_IDENTICAL = 280;
 453 const int UserTokenId_T_IS_IDENTICAL = 281;
 454 const int UserTokenId_T_IS_NOT_EQUAL = 282;
 455 const int UserTokenId_T_IS_EQUAL = 283;
 456 const int UserTokenId_T_IS_GREATER_OR_EQUAL = 284;
 457 const int UserTokenId_T_IS_SMALLER_OR_EQUAL = 285;
 458 const int UserTokenId_T_SR = 286;
 459 const int UserTokenId_T_SL = 287;
 460 const int UserTokenId_T_INSTANCEOF = 288;
 461 const int UserTokenId_T_UNSET_CAST = 289;
 462 const int UserTokenId_T_BOOL_CAST = 290;
 463 const int UserTokenId_T_OBJECT_CAST = 291;
 464 const int UserTokenId_T_ARRAY_CAST = 292;
 465 const int UserTokenId_T_STRING_CAST = 293;
 466 const int UserTokenId_T_DOUBLE_CAST = 294;
 467 const int UserTokenId_T_INT_CAST = 295;
 468 const int UserTokenId_T_DEC = 296;
 469 const int UserTokenId_T_INC = 297;
 470 const int UserTokenId_T_CLONE = 298;
 471 const int UserTokenId_T_NEW = 299;
 472 const int UserTokenId_T_EXIT = 300;
 473 const int UserTokenId_T_IF = 301;
 474 const int UserTokenId_T_ELSEIF = 302;
 475 const int UserTokenId_T_ELSE = 303;
 476 const int UserTokenId_T_ENDIF = 304;
 477 const int UserTokenId_T_LNUMBER = 305;
 478 const int UserTokenId_T_DNUMBER = 306;
 479 const int UserTokenId_T_STRING = 307;
 480 const int UserTokenId_T_STRING_VARNAME = 308;
 481 const int UserTokenId_T_VARIABLE = 309;
 482 const int UserTokenId_T_NUM_STRING = 310;
 483 const int UserTokenId_T_INLINE_HTML = 311;
 484 const int UserTokenId_T_CHARACTER = 312;
 485 const int UserTokenId_T_BAD_CHARACTER = 313;
 486 const int UserTokenId_T_ENCAPSED_AND_WHITESPACE = 314;
 487 const int UserTokenId_T_CONSTANT_ENCAPSED_STRING = 315;
 488 const int UserTokenId_T_ECHO = 316;
 489 const int UserTokenId_T_DO = 317;
 490 const int UserTokenId_T_WHILE = 318;
 491 const int UserTokenId_T_ENDWHILE = 319;
 492 const int UserTokenId_T_FOR = 320;
 493 const int UserTokenId_T_ENDFOR = 321;
 494 const int UserTokenId_T_FOREACH = 322;
 495 const int UserTokenId_T_ENDFOREACH = 323;
 496 const int UserTokenId_T_DECLARE = 324;
 497 const int UserTokenId_T_ENDDECLARE = 325;
 498 const int UserTokenId_T_AS = 326;
 499 const int UserTokenId_T_SWITCH = 327;
 500 const int UserTokenId_T_ENDSWITCH = 328;
 501 const int UserTokenId_T_CASE = 329;
 502 const int UserTokenId_T_DEFAULT = 330;
 503 const int UserTokenId_T_BREAK = 331;
 504 const int UserTokenId_T_GOTO = 332;
 505 const int UserTokenId_T_CONTINUE = 333;
 506 const int UserTokenId_T_FUNCTION = 334;
 507 const int UserTokenId_T_CONST = 335;
 508 const int UserTokenId_T_RETURN = 336;
 509 const int UserTokenId_T_TRY = 337;
 510 const int UserTokenId_T_CATCH = 338;
 511 const int UserTokenId_T_THROW = 339;
 512 const int UserTokenId_T_USE = 340;
 513 const int UserTokenId_T_GLOBAL = 341;
 514 const int UserTokenId_T_PUBLIC = 342;
 515 const int UserTokenId_T_PROTECTED = 343;
 516 const int UserTokenId_T_PRIVATE = 344;
 517 const int UserTokenId_T_FINAL = 345;
 518 const int UserTokenId_T_ABSTRACT = 346;
 519 const int UserTokenId_T_STATIC = 347;
 520 const int UserTokenId_T_VAR = 348;
 521 const int UserTokenId_T_UNSET = 349;
 522 const int UserTokenId_T_ISSET = 350;
 523 const int UserTokenId_T_EMPTY = 351;
 524 const int UserTokenId_T_HALT_COMPILER = 352;
 525 const int UserTokenId_T_CLASS = 353;
 526 const int UserTokenId_T_INTERFACE = 354;
 527 const int UserTokenId_T_EXTENDS = 355;
 528 const int UserTokenId_T_IMPLEMENTS = 356;
 529 const int UserTokenId_T_OBJECT_OPERATOR = 357;
 530 const int UserTokenId_T_DOUBLE_ARROW = 358;
 531 const int UserTokenId_T_LIST = 359;
 532 const int UserTokenId_T_ARRAY = 360;
 533 const int UserTokenId_T_CLASS_C = 361;
 534 const int UserTokenId_T_METHOD_C = 362;
 535 const int UserTokenId_T_FUNC_C = 363;
 536 const int UserTokenId_T_LINE = 364;
 537 const int UserTokenId_T_FILE = 365;
 538 const int UserTokenId_T_COMMENT = 366;
 539 const int UserTokenId_T_DOC_COMMENT = 367;
 540 const int UserTokenId_T_OPEN_TAG = 368;
 541 const int UserTokenId_T_OPEN_TAG_WITH_ECHO = 369;
 542 const int UserTokenId_T_CLOSE_TAG = 370;
 543 const int UserTokenId_T_WHITESPACE = 371;
 544 const int UserTokenId_T_START_HEREDOC = 372;
 545 const int UserTokenId_T_END_HEREDOC = 373;
 546 const int UserTokenId_T_DOLLAR_OPEN_CURLY_BRACES = 374;
 547 const int UserTokenId_T_CURLY_OPEN = 375;
 548 const int UserTokenId_T_PAAMAYIM_NEKUDOTAYIM UNUSED = 376;
 549 const int UserTokenId_T_NAMESPACE = 377;
 550 const int UserTokenId_T_NS_C = 378;
 551 const int UserTokenId_T_DIR = 379;
 552 const int UserTokenId_T_NS_SEPARATOR = 380;
 553 const int UserTokenId_T_YIELD = 381;
 554 const int UserTokenId_T_XHP_LABEL = 382;
 555 const int UserTokenId_T_XHP_TEXT = 383;
 556 const int UserTokenId_T_XHP_ATTRIBUTE = 384;
 557 const int UserTokenId_T_XHP_CATEGORY = 385;
 558 const int UserTokenId_T_XHP_CATEGORY_LABEL = 386;
 559 const int UserTokenId_T_XHP_CHILDREN = 387;
 560 const int UserTokenId_T_XHP_ENUM = 388;
 561 const int UserTokenId_T_XHP_REQUIRED = 389;
 562 const int UserTokenId_T_TRAIT = 390;
 563 const int UserTokenId_T_INSTEADOF = 391;
 564 const int UserTokenId_T_TRAIT_C = 392;
 565 const int UserTokenId_T_ELLIPSIS = 393;
 566 const int UserTokenId_T_HH_ERROR = 394;
 567 const int UserTokenId_T_FINALLY = 395;
 568 const int UserTokenId_T_XHP_TAG_LT = 396;
 569 const int UserTokenId_T_XHP_TAG_GT = 397;
 570 const int UserTokenId_T_TYPELIST_LT = 398;
 571 const int UserTokenId_T_TYPELIST_GT = 399;
 572 const int UserTokenId_T_UNRESOLVED_LT = 400;
 573 const int UserTokenId_T_COLLECTION = 401;
 574 const int UserTokenId_T_SHAPE = 402;
 575 const int UserTokenId_T_TYPE = 403;
 576 const int UserTokenId_T_UNRESOLVED_TYPE = 404;
 577 const int UserTokenId_T_NEWTYPE = 405;
 578 const int UserTokenId_T_UNRESOLVED_NEWTYPE = 406;
 579 const int UserTokenId_T_COMPILER_HALT_OFFSET = 407;
 580 const int UserTokenId_T_AWAIT = 408;
 581 const int UserTokenId_T_ASYNC = 409;
 582 const int UserTokenId_T_FROM = 411;
 583 const int UserTokenId_T_WHERE = 412;
 584 const int UserTokenId_T_JOIN = 413;
 585 const int UserTokenId_T_IN = 414;
 586 const int UserTokenId_T_ON = 415;
 587 const int UserTokenId_T_EQUALS = 416;
 588 const int UserTokenId_T_INTO = 417;
 589 const int UserTokenId_T_LET = 418;
 590 const int UserTokenId_T_ORDERBY = 419;
 591 const int UserTokenId_T_ASCENDING = 420;
 592 const int UserTokenId_T_DESCENDING = 421;
 593 const int UserTokenId_T_SELECT = 422;
 594 const int UserTokenId_T_GROUP = 423;
 595 const int UserTokenId_T_BY = 424;
 596 const int UserTokenId_T_LAMBDA_ARROW = 425;
 597 const int UserTokenId_T_DOUBLE_COLON = 426;
 598 const int UserTokenId_T_LAMBDA_OP = 427;
 599 const int UserTokenId_T_LAMBDA_CP = 428;
 600 const int UserTokenId_T_UNRESOLVED_OP = 429;
 601 const int UserTokenId_T_CALLABLE = 430;
 602 const int UserTokenId_T_ONUMBER = 431;
 603 const int UserTokenId_T_POW = 432;
 604 const int UserTokenId_T_POW_EQUAL = 433;
 605 const int MaxUserTokenId = 434; // Marker, not a real user token ID
 606
 607 #undef YYTOKENTYPE
 608 #undef YYTOKEN_MAP
 609 #undef YYTOKEN
 610 #define YYTOKEN(num, name) UserTokenId_##name,
 611 #define YYTOKEN_MAP static const int user_token_ids[] =
 612 #include "hphp/parser/hphp.tab.hpp"
 613 #undef YYTOKEN_MAP
 614 #undef YYTOKEN
 615
 616 // Converts an internal token ID to a user token ID
 617 static int get_user_token_id(int internal_id) {
 618   assert(internal_id >= 0);
 619   if (internal_id < 256) {
 620     return internal_id;
 621   }
 622   if (internal_id >= YYTOKEN_MIN && internal_id <= YYTOKEN_MAX) {
 623     return user_token_ids[internal_id - YYTOKEN_MIN];
 624   }
 625   return MaxUserTokenId;
 626 }
 627
 628 Array f_token_get_all(const String& source) {
 629   Scanner scanner(source.data(), source.size(),
 630                   RuntimeOption::GetScannerType() | Scanner::ReturnAllTokens);
 631   ScannerToken tok;
 632   Location loc;
 633   int tokid;
 634   Array res;
 635   while ((tokid = scanner.getNextToken(tok, loc))) {
 636     if (tokid < 256) {
 637       res.append(String::FromChar((char)tokid));
 638     } else {
 639       String value;
 640       const int tokVal = get_user_token_id(tokid);
 641       if (tokVal == UserTokenId_T_XHP_LABEL) {
 642         value = String(":" + tok.text());
 643       } else if (tokVal == UserTokenId_T_XHP_CATEGORY_LABEL) {
 644         value = String("%" + tok.text());
 645       } else {
 646         value = String(tok.text());
 647       }
 648       Array p = make_packed_array(
 649         // Convert the internal token ID to a user token ID
 650         tokVal,
 651         value,
 652         loc.line0
 653       );
 654       res.append(p);
 655     }
 656   }
 657   return res;
 658 }
 659
 660 // User token ID => Token name mapping
 661 static const char** getTokenNameTable() {
 662   static const char* table[MaxUserTokenId+1];
 663   for (int i = 0; i <= MaxUserTokenId; ++i) {
 664     table[i] = "UNKNOWN";
 665   }
 666 #undef YYTOKENTYPE
 667 #undef YYTOKEN_MAP
 668 #undef YYTOKEN
 669 #define YYTOKEN(num, name) table[get_user_token_id(num)] = #name;
 670 #define YYTOKEN_MAP
 671 #include "hphp/parser/hphp.tab.hpp"
 672 #undef YYTOKEN_MAP
 673 #undef YYTOKEN
 674   return table;
 675 }
 676
 677 // Converts a user token ID to a token name
 678 String f_token_name(int64_t token) {
 679   static const char** table = getTokenNameTable();
 680   // For compatibility with parser packages expecting veneration of the
 681   // lexer's Hebrew roots.
 682   if (token == k_T_DOUBLE_COLON) {
 683     return "T_PAAMAYIM_NEKUDOTAYIM";
 684   }
 685   if (token >= 0 && token < MaxUserTokenId) {
 686     return table[token];
 687   }
 688   return "UNKNOWN";
 689 }
 690
 691 String f_hphp_to_string(const Variant& v) {
 692   return v.toString();
 693 }
 694
 695 ///////////////////////////////////////////////////////////////////////////////
 696 }
 697
 698 #undef YYTOKENTYPE
 699 #undef YYTOKEN_MAP
 700 #undef YYTOKEN
 701 #define YYTOKEN(num, name)                      \
 702   extern const int64_t k_##name = get_user_token_id(num);
 703 #define YYTOKEN_MAP namespace HPHP
 704
 705 #include "hphp/parser/hphp.tab.hpp"
 706
 707 namespace HPHP {
 708 extern const int64_t k_T_PAAMAYIM_NEKUDOTAYIM = k_T_DOUBLE_COLON;
 709 }
 710
 711 #undef YYTOKEN_MAP
 712 #undef YYTOKEN