hphp/runtime/vm/jit/irlower-minstr.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/runtime/vm/jit/irlower-internal.h"
  18
  19 #include "hphp/runtime/base/array-data.h"
  20 #include "hphp/runtime/base/header-kind.h"
  21 #include "hphp/runtime/base/mixed-array.h"
  22 #include "hphp/runtime/base/packed-array.h"
  23 #include "hphp/runtime/base/rds.h"
  24 #include "hphp/runtime/base/typed-value.h"
  25 #include "hphp/runtime/vm/member-operations.h"
  26 #include "hphp/runtime/vm/unit.h"
  27
  28 #include "hphp/runtime/vm/jit/abi.h"
  29 #include "hphp/runtime/vm/jit/arg-group.h"
  30 #include "hphp/runtime/vm/jit/array-offset-profile.h"
  31 #include "hphp/runtime/vm/jit/bc-marker.h"
  32 #include "hphp/runtime/vm/jit/code-gen-cf.h"
  33 #include "hphp/runtime/vm/jit/code-gen-helpers.h"
  34 #include "hphp/runtime/vm/jit/code-gen-internal.h"
  35 #include "hphp/runtime/vm/jit/ir-instruction.h"
  36 #include "hphp/runtime/vm/jit/minstr-helpers.h"
  37 #include "hphp/runtime/vm/jit/translator-inline.h"
  38
  39 #include "hphp/util/immed.h"
  40 #include "hphp/util/stack-trace.h"
  41 #include "hphp/util/struct-log.h"
  42 #include "hphp/util/trace.h"
  43
  44 // This file does ugly things with macros so include last.
  45 #include "hphp/runtime/vm/jit/irlower-minstr-internal.h"
  46
  47 namespace HPHP { namespace jit { namespace irlower {
  48
  49 TRACE_SET_MOD(irlower);
  50
  51 ///////////////////////////////////////////////////////////////////////////////
  52
  53 void cgBaseG(IRLS& env, const IRInstruction* inst) {
  54   auto const mode = inst->extra<MOpModeData>()->mode;
  55   BUILD_OPTAB(BASE_G_HELPER_TABLE, mode);
  56
  57   auto const args = argGroup(env, inst).typedValue(0);
  58
  59   auto& v = vmain(env);
  60   cgCallHelper(v, env, CallSpec::direct(opFunc), callDest(env, inst),
  61                SyncOptions::Sync, args);
  62 }
  63
  64 void cgFinishMemberOp(IRLS&, const IRInstruction*) {}
  65
  66 ///////////////////////////////////////////////////////////////////////////////
  67
  68 namespace {
  69
  70 /*
  71  * Make an ArgGroup for prop instructions that takes:
  72  *    1/ the context Class*
  73  *    2/ the ObjectData* (or pointer to a KindOfObject TV)
  74  */
  75 ArgGroup propArgs(IRLS& env, const IRInstruction* inst) {
  76   return argGroup(env, inst)
  77     .immPtr(inst->marker().func()->cls())
  78     .ssa(0);
  79 }
  80
  81 void implProp(IRLS& env, const IRInstruction* inst) {
  82   auto const mode    = inst->extra<MOpModeData>()->mode;
  83   auto const base    = inst->src(0);
  84   auto const key     = inst->src(1);
  85   auto const keyType = getKeyTypeNoInt(key);
  86
  87   void (*helper)();
  88   if (base->isA(TObj)) {
  89     BUILD_OPTAB(PROP_OBJ_HELPER_TABLE, mode, keyType);
  90     helper = opFunc;
  91   } else {
  92     BUILD_OPTAB(PROP_HELPER_TABLE, mode, keyType);
  93     helper = opFunc;
  94   }
  95
  96   auto const args = propArgs(env, inst)
  97     .memberKeyS(1)
  98     .ssa(2);
  99
 100   auto& v = vmain(env);
 101   cgCallHelper(v, env, CallSpec::direct(helper), callDest(env, inst),
 102                SyncOptions::Sync, args);
 103 }
 104
 105 void implIssetEmptyProp(IRLS& env, const IRInstruction* inst) {
 106   auto const isEmpty = inst->op() == EmptyProp;
 107   auto const base = inst->src(0);
 108   auto const key = inst->src(1);
 109   auto const keyType = getKeyTypeNoInt(key);
 110
 111   void (*helper)();
 112   if (base->isA(TObj)) {
 113     BUILD_OPTAB(ISSET_EMPTY_OBJ_PROP_HELPER_TABLE, keyType, isEmpty);
 114     helper = opFunc;
 115   } else {
 116     BUILD_OPTAB(ISSET_EMPTY_PROP_HELPER_TABLE, keyType, isEmpty);
 117     helper = opFunc;
 118   }
 119
 120   auto const args = propArgs(env, inst).memberKeyS(1);
 121
 122   auto& v = vmain(env);
 123   cgCallHelper(v, env, CallSpec::direct(helper), callDest(env, inst),
 124                SyncOptions::Sync, args);
 125 }
 126
 127 }
 128
 129 ///////////////////////////////////////////////////////////////////////////////
 130
 131 void cgPropX(IRLS& env, const IRInstruction* i)  { implProp(env, i); }
 132 void cgPropDX(IRLS& env, const IRInstruction* i) { implProp(env, i); }
 133
 134 void cgPropQ(IRLS& env, const IRInstruction* inst) {
 135   using namespace MInstrHelpers;
 136
 137   auto const args = propArgs(env, inst).ssa(1).ssa(2);
 138
 139   auto helper = inst->src(0)->isA(TObj)
 140     ? CallSpec::direct(propCOQ)
 141     : CallSpec::direct(propCQ);
 142
 143   auto& v = vmain(env);
 144   cgCallHelper(v, env, helper, callDest(env, inst), SyncOptions::Sync, args);
 145 }
 146
 147 void cgCGetProp(IRLS& env, const IRInstruction* inst) {
 148   auto const mode    = inst->extra<MOpModeData>()->mode;
 149   auto const base    = inst->src(0);
 150   auto const key     = inst->src(1);
 151   auto const keyType = getKeyTypeNoInt(key);
 152
 153   void (*helper)();
 154   if (base->isA(TObj)) {
 155     BUILD_OPTAB(CGET_OBJ_PROP_HELPER_TABLE, keyType, mode);
 156     helper = opFunc;
 157   } else {
 158     BUILD_OPTAB(CGET_PROP_HELPER_TABLE, keyType, mode);
 159     helper = opFunc;
 160   }
 161
 162   auto const args = propArgs(env, inst).memberKeyS(1);
 163
 164   auto& v = vmain(env);
 165   cgCallHelper(v, env, CallSpec::direct(helper), callDestTV(env, inst),
 166                SyncOptions::Sync, args);
 167 }
 168
 169 void cgCGetPropQ(IRLS& env, const IRInstruction* inst) {
 170   using namespace MInstrHelpers;
 171
 172   auto args = propArgs(env, inst).ssa(1);
 173
 174   auto& v = vmain(env);
 175
 176   if (inst->src(0)->isA(TObj)) {
 177     return cgCallHelper(v, env, CallSpec::direct(cGetPropSOQ),
 178                         callDestTV(env, inst), SyncOptions::Sync, args);
 179   }
 180   cgCallHelper(v, env, CallSpec::direct(cGetPropSQ),
 181                callDestTV(env, inst), SyncOptions::Sync, args);
 182 }
 183
 184 void cgVGetProp(IRLS& env, const IRInstruction* inst) {
 185   auto const base    = inst->src(0);
 186   auto const key     = inst->src(1);
 187   auto const keyType = getKeyTypeNoInt(key);
 188
 189   void (*helper)();
 190   if (base->isA(TObj)) {
 191     BUILD_OPTAB(VGET_OBJ_PROP_HELPER_TABLE, keyType);
 192     helper = opFunc;
 193   } else {
 194     BUILD_OPTAB(VGET_PROP_HELPER_TABLE, keyType);
 195     helper = opFunc;
 196   }
 197
 198   auto const args = propArgs(env, inst).memberKeyS(1);
 199
 200   auto& v = vmain(env);
 201   cgCallHelper(v, env, CallSpec::direct(helper), callDest(env, inst),
 202                SyncOptions::Sync, args);
 203 }
 204
 205 void cgBindProp(IRLS& env, const IRInstruction* inst) {
 206   auto const base = inst->src(0);
 207
 208   auto helper = base->isA(TObj)
 209     ? CallSpec::direct(MInstrHelpers::bindPropCO)
 210     : CallSpec::direct(MInstrHelpers::bindPropC);
 211
 212   auto const args = propArgs(env, inst).typedValue(1).ssa(2);
 213
 214   auto& v = vmain(env);
 215   cgCallHelper(v, env, helper, callDest(env, inst), SyncOptions::Sync, args);
 216 }
 217
 218 void cgSetProp(IRLS& env, const IRInstruction* inst) {
 219   auto const base = inst->src(0);
 220   auto const key = inst->src(1);
 221   auto const keyType = getKeyTypeNoInt(key);
 222
 223   void (*helper)();
 224   if (base->isA(TObj)) {
 225     BUILD_OPTAB(SETPROP_OBJ_HELPER_TABLE, keyType);
 226     helper = opFunc;
 227   } else {
 228     BUILD_OPTAB(SETPROP_HELPER_TABLE, keyType);
 229     helper = opFunc;
 230   }
 231
 232   auto const args = propArgs(env, inst)
 233     .memberKeyS(1)
 234     .typedValue(2);
 235
 236   auto& v = vmain(env);
 237   cgCallHelper(v, env, CallSpec::direct(helper),
 238                kVoidDest, SyncOptions::Sync, args);
 239 }
 240
 241 void cgUnsetProp(IRLS& env, const IRInstruction* inst) {
 242   auto const base = inst->src(0);
 243
 244   auto helper = base->isA(TObj)
 245     ? CallSpec::direct(MInstrHelpers::unsetPropCO)
 246     : CallSpec::direct(MInstrHelpers::unsetPropC);
 247
 248   auto const args = propArgs(env, inst).typedValue(1);
 249
 250   auto& v = vmain(env);
 251   cgCallHelper(v, env, helper, kVoidDest, SyncOptions::Sync, args);
 252 }
 253
 254 void cgSetOpProp(IRLS& env, const IRInstruction* inst) {
 255   auto const base = inst->src(0);
 256   auto const extra = inst->extra<SetOpProp>();
 257
 258   auto helper = base->isA(TObj)
 259     ? CallSpec::direct(MInstrHelpers::setOpPropCO)
 260     : CallSpec::direct(MInstrHelpers::setOpPropC);
 261
 262   auto const args = propArgs(env, inst)
 263     .typedValue(1)
 264     .typedValue(2)
 265     .imm(static_cast<int32_t>(extra->op));
 266
 267   auto& v = vmain(env);
 268   cgCallHelper(v, env, helper, callDestTV(env, inst), SyncOptions::Sync, args);
 269 }
 270
 271 void cgIncDecProp(IRLS& env, const IRInstruction* inst) {
 272   auto const base = inst->src(0);
 273   auto const extra = inst->extra<IncDecProp>();
 274
 275   auto helper = base->isA(TObj)
 276     ? CallSpec::direct(MInstrHelpers::incDecPropCO)
 277     : CallSpec::direct(MInstrHelpers::incDecPropC);
 278
 279   auto const args = propArgs(env, inst)
 280     .typedValue(1)
 281     .imm(static_cast<int32_t>(extra->op));
 282
 283   auto& v = vmain(env);
 284   cgCallHelper(v, env, helper, callDestTV(env, inst),
 285                SyncOptions::Sync, args);
 286 }
 287
 288 void cgIssetProp(IRLS& env, const IRInstruction* i) {
 289   implIssetEmptyProp(env, i);
 290 }
 291 void cgEmptyProp(IRLS& env, const IRInstruction* i) {
 292   implIssetEmptyProp(env, i);
 293 }
 294
 295 ///////////////////////////////////////////////////////////////////////////////
 296
 297 namespace {
 298
 299 /*
 300  * Make an ArgGroup for elem instructions that takes:
 301  *    1/ the pointer to a KindOfArray TV
 302  *    2/ the index key, as a raw value or a TypedValue depending on whether
 303  *       the type is known
 304  */
 305 ArgGroup elemArgs(IRLS& env, const IRInstruction* inst) {
 306   return argGroup(env, inst)
 307     .ssa(0)
 308     .memberKeyIS(1);
 309 }
 310
 311 void implElem(IRLS& env, const IRInstruction* inst) {
 312   auto const mode  = inst->extra<MOpModeData>()->mode;
 313   auto const key   = inst->src(1);
 314   BUILD_OPTAB(ELEM_HELPER_TABLE, getKeyType(key), mode, checkHACIntishCast());
 315
 316   auto const args = elemArgs(env, inst).ssa(2);
 317
 318   auto& v = vmain(env);
 319   cgCallHelper(v, env, CallSpec::direct(opFunc), callDest(env, inst),
 320                SyncOptions::Sync, args);
 321 }
 322
 323 void implIssetEmptyElem(IRLS& env, const IRInstruction* inst) {
 324   auto const isEmpty = inst->op() == EmptyElem;
 325   auto const key     = inst->src(1);
 326   BUILD_OPTAB(ISSET_EMPTY_ELEM_HELPER_TABLE, getKeyType(key),
 327               isEmpty, checkHACIntishCast());
 328
 329   auto& v = vmain(env);
 330   cgCallHelper(v, env, CallSpec::direct(opFunc), callDest(env, inst),
 331                SyncOptions::Sync, elemArgs(env, inst));
 332 }
 333
 334 }
 335
 336 ///////////////////////////////////////////////////////////////////////////////
 337
 338 void cgElemX(IRLS& env, const IRInstruction* i)  { implElem(env, i); }
 339 void cgElemDX(IRLS& env, const IRInstruction* i) { implElem(env, i); }
 340 void cgElemUX(IRLS& env, const IRInstruction* i) { implElem(env, i); }
 341
 342 void cgCGetElem(IRLS& env, const IRInstruction* inst) {
 343   auto const mode  = inst->extra<MOpModeData>()->mode;
 344   auto const key   = inst->src(1);
 345   BUILD_OPTAB(CGETELEM_HELPER_TABLE, getKeyType(key), mode,
 346               checkHACIntishCast());
 347
 348   auto& v = vmain(env);
 349   cgCallHelper(v, env, CallSpec::direct(opFunc), callDestTV(env, inst),
 350                SyncOptions::Sync, elemArgs(env, inst));
 351 }
 352
 353 void cgVGetElem(IRLS& env, const IRInstruction* inst) {
 354   auto const key = inst->src(1);
 355   BUILD_OPTAB(VGETELEM_HELPER_TABLE, getKeyType(key),
 356               checkHACIntishCast());
 357
 358   auto& v = vmain(env);
 359   cgCallHelper(v, env, CallSpec::direct(opFunc), callDest(env, inst),
 360                SyncOptions::Sync, elemArgs(env, inst));
 361 }
 362
 363 void cgSetElem(IRLS& env, const IRInstruction* inst) {
 364   auto const key = inst->src(1);
 365   BUILD_OPTAB(SETELEM_HELPER_TABLE, getKeyType(key),
 366               checkHACIntishCast());
 367
 368   auto& v = vmain(env);
 369   cgCallHelper(v, env, CallSpec::direct(opFunc), callDest(env, inst),
 370                SyncOptions::Sync, elemArgs(env, inst).typedValue(2));
 371 }
 372
 373 IMPL_OPCODE_CALL(SetNewElem);
 374 IMPL_OPCODE_CALL(BindNewElem);
 375
 376 void cgSetWithRefElem(IRLS& env, const IRInstruction* inst) {
 377   auto& v = vmain(env);
 378
 379   auto const target = checkHACIntishCast()
 380     ? CallSpec::direct(MInstrHelpers::setWithRefElem<true>)
 381     : CallSpec::direct(MInstrHelpers::setWithRefElem<false>);
 382
 383   auto const args = argGroup(env, inst)
 384     .ssa(0)
 385     .typedValue(1)
 386     .typedValue(2);
 387
 388   cgCallHelper(v, env, target, kVoidDest, SyncOptions::Sync, args);
 389 }
 390
 391 void cgBindElem(IRLS& env, const IRInstruction* inst) {
 392   auto& v = vmain(env);
 393
 394   auto const target = checkHACIntishCast()
 395     ? CallSpec::direct(MInstrHelpers::bindElemC<true>)
 396     : CallSpec::direct(MInstrHelpers::bindElemC<false>);
 397
 398   auto const args = argGroup(env, inst)
 399     .ssa(0)
 400     .typedValue(1)
 401     .ssa(2);
 402
 403   cgCallHelper(v, env, target, kVoidDest, SyncOptions::Sync, args);
 404 }
 405
 406 void cgSetOpElem(IRLS& env, const IRInstruction* inst) {
 407   auto& v = vmain(env);
 408
 409   auto const target = checkHACIntishCast()
 410     ? CallSpec::direct(MInstrHelpers::setOpElem<true>)
 411     : CallSpec::direct(MInstrHelpers::setOpElem<false>);
 412
 413   auto const args = argGroup(env, inst)
 414     .ssa(0)
 415     .typedValue(1)
 416     .typedValue(2)
 417     .imm(uint32_t(inst->extra<SetOpElem>()->op));
 418
 419   cgCallHelper(v, env, target, callDestTV(env, inst),
 420                SyncOptions::Sync, args);
 421 }
 422
 423 void cgIncDecElem(IRLS& env, const IRInstruction* inst) {
 424   auto& v = vmain(env);
 425
 426   auto const target = checkHACIntishCast()
 427     ? CallSpec::direct(MInstrHelpers::incDecElem<true>)
 428     : CallSpec::direct(MInstrHelpers::incDecElem<false>);
 429
 430   auto const args = argGroup(env, inst)
 431     .ssa(0)
 432     .typedValue(1)
 433     .imm(uint32_t(inst->extra<IncDecElem>()->op));
 434
 435   cgCallHelper(v, env, target, callDestTV(env, inst),
 436                SyncOptions::Sync, args);
 437 }
 438
 439 void cgUnsetElem(IRLS& env, const IRInstruction* inst) {
 440   auto const key = inst->src(1);
 441   BUILD_OPTAB(UNSET_ELEM_HELPER_TABLE, getKeyType(key),
 442               checkHACIntishCast());
 443
 444   auto& v = vmain(env);
 445   cgCallHelper(v, env, CallSpec::direct(opFunc), kVoidDest,
 446                SyncOptions::Sync, elemArgs(env, inst));
 447 }
 448
 449 void cgIssetElem(IRLS& env, const IRInstruction* i) {
 450   implIssetEmptyElem(env, i);
 451 }
 452 void cgEmptyElem(IRLS& env, const IRInstruction* i) {
 453   implIssetEmptyElem(env, i);
 454 }
 455
 456 ///////////////////////////////////////////////////////////////////////////////
 457
 458 namespace {
 459
 460 /*
 461  * Make an ArgGroup for array elem instructions that takes:
 462  *    1/ the ArrayData* (or pointer to a KindOfArray TV)
 463  *    2/ the index key
 464  */
 465 ArgGroup arrArgs(IRLS& env, const IRInstruction* inst,
 466                  const ArrayKeyInfo& keyInfo) {
 467   auto args = argGroup(env, inst).ssa(0);
 468   if (keyInfo.converted) {
 469     args.imm(keyInfo.convertedInt);
 470   } else {
 471     args.ssa(1);
 472   }
 473   return args;
 474 }
 475
 476 }
 477
 478 ///////////////////////////////////////////////////////////////////////////////
 479 // Offset profiling.
 480
 481 namespace {
 482
 483 template<class OpFunc>
 484 void implProfileHackArrayOffset(IRLS& env, const IRInstruction* inst,
 485                                 OpFunc opFunc) {
 486   auto& v = vmain(env);
 487
 488   auto const rprof = v.makeReg();
 489   v << lea{rvmtl()[inst->extra<RDSHandleData>()->handle], rprof};
 490
 491   auto args = argGroup(env, inst).ssa(0).ssa(1).reg(rprof);
 492
 493   cgCallHelper(v, env, CallSpec::direct(opFunc), kVoidDest,
 494                SyncOptions::Sync, args);
 495 }
 496
 497 void implCheckMixedArrayLikeOffset(IRLS& env, const IRInstruction* inst,
 498                                    KeyType key_type) {
 499   auto const arr = srcLoc(env, inst, 0).reg();
 500   auto const key = srcLoc(env, inst, 1).reg();
 501   auto const branch = label(env, inst->taken());
 502   auto const pos = inst->extra<IndexData>()->index;
 503   auto& v = vmain(env);
 504
 505   auto const elmOff = MixedArray::elmOff(pos);
 506   using Elm = MixedArray::Elm;
 507
 508   { // Also fail if our predicted position exceeds bounds.
 509     auto const sf = v.makeReg();
 510     v << cmplim{safe_cast<int32_t>(pos), arr[MixedArray::usedOff()], sf};
 511     ifThen(v, CC_LE, sf, branch);
 512   }
 513   { // Fail if the Elm key value doesn't match.
 514     auto const sf = v.makeReg();
 515     v << cmpqm{key, arr[elmOff + Elm::keyOff()], sf};
 516     ifThen(v, CC_NE, sf, branch);
 517   }
 518   { // Fail if the Elm key type doesn't match.
 519     auto const sf = v.makeReg();
 520     v << cmplim{0, arr[elmOff + Elm::dataOff() + TVOFF(m_aux)], sf};
 521
 522     assertx(key_type != KeyType::Any);
 523
 524     // Note that if `key' actually is an integer-ish string, we'd fail this
 525     // check (and most likely would have failed the previous check also), but
 526     // this false negative is allowed.
 527     auto const is_str_key = key_type == KeyType::Str;
 528     ifThen(v, is_str_key ? CC_L : CC_GE, sf, branch);
 529   }
 530   { // Fail if the Elm is a tombstone.  See MixedArray::isTombstone().
 531     auto const sf = v.makeReg();
 532     v << cmpbim{KindOfUninit, arr[elmOff + Elm::dataOff() + TVOFF(m_type)], sf};
 533     ifThen(v, CC_L, sf, branch);
 534   }
 535 }
 536
 537 }
 538
 539 void cgProfileMixedArrayOffset(IRLS& env, const IRInstruction* inst) {
 540   auto const arr = inst->src(0);
 541   auto const key = inst->src(1);
 542   auto const keyInfo = checkStrictlyInteger(arr->type(), key->type());
 543
 544   BUILD_OPTAB(PROFILE_MIXED_ARRAY_OFFSET_HELPER_TABLE,
 545               keyInfo.type,
 546               keyInfo.checkForInt);
 547   auto& v = vmain(env);
 548
 549   auto const rprof = v.makeReg();
 550   v << lea{rvmtl()[inst->extra<ProfileMixedArrayOffset>()->handle], rprof};
 551
 552   cgCallHelper(v, env, CallSpec::direct(opFunc), kVoidDest, SyncOptions::Sync,
 553                arrArgs(env, inst, keyInfo).reg(rprof));
 554 }
 555
 556 void cgProfileDictOffset(IRLS& env, const IRInstruction* inst) {
 557   BUILD_OPTAB(PROFILE_DICT_OFFSET_HELPER_TABLE, getKeyType(inst->src(1)));
 558   implProfileHackArrayOffset(env, inst, opFunc);
 559 }
 560
 561 void cgProfileKeysetOffset(IRLS& env, const IRInstruction* inst) {
 562   BUILD_OPTAB(PROFILE_KEYSET_OFFSET_HELPER_TABLE, getKeyType(inst->src(1)));
 563   implProfileHackArrayOffset(env, inst, opFunc);
 564 }
 565
 566 void cgCheckMixedArrayOffset(IRLS& env, const IRInstruction* inst) {
 567   implCheckMixedArrayLikeOffset(
 568     env, inst,
 569     checkStrictlyInteger(inst->src(0)->type(), inst->src(1)->type()).type
 570   );
 571 }
 572
 573 void cgCheckDictOffset(IRLS& env, const IRInstruction* inst) {
 574   implCheckMixedArrayLikeOffset(env, inst, getKeyType(inst->src(1)));
 575 }
 576
 577 void cgCheckKeysetOffset(IRLS& env, const IRInstruction* inst) {
 578   auto const keyset = srcLoc(env, inst, 0).reg();
 579   auto const key = srcLoc(env, inst, 1).reg();
 580   auto const branch = label(env, inst->taken());
 581   auto const pos = inst->extra<CheckKeysetOffset>()->index;
 582   auto& v = vmain(env);
 583   auto const tvOff = SetArray::tvOff(pos);
 584
 585   { // Fail if our predicted position exceeds bounds.
 586     auto const sf = v.makeReg();
 587     v << cmplim{safe_cast<int32_t>(pos), keyset[SetArray::usedOff()], sf};
 588     ifThen(v, CC_LE, sf, branch);
 589   }
 590   { // Fail if the Elm key value doesn't match.
 591     auto const sf = v.makeReg();
 592     v << cmpqm{key, keyset[tvOff + TVOFF(m_data)], sf};
 593     ifThen(v, CC_NE, sf, branch);
 594   }
 595   { // Fail if the Elm key type doesn't match.
 596     auto const sf = v.makeReg();
 597     v << cmplim{0, keyset[tvOff + TVOFF(m_aux)], sf};
 598
 599     auto const key_type = getKeyType(inst->src(1));
 600     assertx(key_type != KeyType::Any);
 601
 602     auto const is_str_key = key_type == KeyType::Str;
 603     ifThen(v, is_str_key ? CC_L : CC_GE, sf, branch);
 604   }
 605   { // Fail if the Elm is a tombstone.  See SetArray::isTombstone().
 606     auto const sf = v.makeReg();
 607     v << cmpbim{KindOfUninit, keyset[tvOff + TVOFF(m_type)], sf};
 608     ifThen(v, CC_L, sf, branch);
 609   }
 610 }
 611
 612 void cgCheckArrayCOW(IRLS& env, const IRInstruction* inst) {
 613   auto const arr = srcLoc(env, inst, 0).reg();
 614   auto& v = vmain(env);
 615
 616   auto const sf = emitCmpRefCount(v, OneReference, arr);
 617   ifThen(v, CC_NE, sf, label(env, inst->taken()));
 618 }
 619
 620 ///////////////////////////////////////////////////////////////////////////////
 621 // Array.
 622
 623 namespace {
 624
 625 void implArraySet(IRLS& env, const IRInstruction* inst) {
 626   bool const setRef  = inst->op() == ArraySetRef;
 627   auto const arr     = inst->src(0);
 628   auto const key     = inst->src(1);
 629   auto const keyInfo = checkStrictlyInteger(arr->type(), key->type());
 630   BUILD_OPTAB(ARRAYSET_HELPER_TABLE,
 631               keyInfo.type,
 632               keyInfo.checkForInt,
 633               setRef,
 634               checkHACIntishCast());
 635
 636
 637   auto args = arrArgs(env, inst, keyInfo);
 638   args.typedValue(2);
 639   if (setRef) args.ssa(3);
 640
 641   auto& v = vmain(env);
 642   cgCallHelper(v, env, CallSpec::direct(opFunc), callDest(env, inst),
 643                SyncOptions::Sync, args);
 644 }
 645
 646 }
 647
 648 void cgElemArrayX(IRLS& env, const IRInstruction* inst) {
 649   auto const arr = inst->src(0);
 650   auto const key = inst->src(1);
 651   auto const mode = inst->extra<ElemArrayX>()->mode;
 652   auto const keyInfo = checkStrictlyInteger(arr->type(), key->type());
 653   BUILD_OPTAB(ELEM_ARRAY_HELPER_TABLE,
 654               keyInfo.type, keyInfo.checkForInt, mode,
 655               checkHACIntishCast());
 656
 657   auto& v = vmain(env);
 658   cgCallHelper(v, env, CallSpec::direct(opFunc), callDest(env, inst),
 659                SyncOptions::Sync, arrArgs(env, inst, keyInfo));
 660 }
 661
 662 void cgElemArrayD(IRLS& env, const IRInstruction* inst) {
 663   auto const key     = inst->src(1);
 664   auto const keyInfo = checkStrictlyInteger(inst->typeParam(), key->type());
 665   BUILD_OPTAB(ELEM_ARRAY_D_HELPER_TABLE, keyInfo.type, checkHACIntishCast());
 666
 667   auto& v = vmain(env);
 668   cgCallHelper(v, env, CallSpec::direct(opFunc), callDest(env, inst),
 669                SyncOptions::Sync, arrArgs(env, inst, keyInfo));
 670 }
 671
 672 void cgElemArrayU(IRLS& env, const IRInstruction* inst) {
 673   auto const key     = inst->src(1);
 674   auto const keyInfo = checkStrictlyInteger(inst->typeParam(), key->type());
 675   BUILD_OPTAB(ELEM_ARRAY_U_HELPER_TABLE, keyInfo.type, checkHACIntishCast());
 676
 677   auto& v = vmain(env);
 678   cgCallHelper(v, env, CallSpec::direct(opFunc), callDest(env, inst),
 679                SyncOptions::Sync, arrArgs(env, inst, keyInfo));
 680 }
 681
 682 void cgElemMixedArrayK(IRLS& env, const IRInstruction* inst) {
 683   auto const arr = srcLoc(env, inst, 0).reg();
 684   auto const dst = dstLoc(env, inst, 0);
 685   auto const pos = inst->extra<ElemMixedArrayK>()->index;
 686   auto const off = MixedArray::elmOff(pos) + MixedArray::Elm::dataOff();
 687
 688   auto& v = vmain(env);
 689
 690   assertx(dst.numAllocated() == 1);
 691   v << lea{arr[off], dst.reg()};
 692 }
 693
 694 void cgArrayGet(IRLS& env, const IRInstruction* inst) {
 695   auto const arr = inst->src(0);
 696   auto const key = inst->src(1);
 697   auto const mode = inst->extra<ArrayGet>()->mode;
 698   auto const keyInfo = checkStrictlyInteger(arr->type(), key->type());
 699   BUILD_OPTAB(ARRAYGET_HELPER_TABLE, keyInfo.type, keyInfo.checkForInt, mode,
 700                       checkHACIntishCast());
 701
 702   auto& v = vmain(env);
 703   cgCallHelper(v, env, CallSpec::direct(opFunc), callDestTV(env, inst),
 704                SyncOptions::Sync, arrArgs(env, inst, keyInfo));
 705 }
 706
 707 void cgMixedArrayGetK(IRLS& env, const IRInstruction* inst) {
 708   auto const arr = srcLoc(env, inst, 0).reg();
 709   auto const pos = inst->extra<MixedArrayGetK>()->index;
 710   auto const off = MixedArray::elmOff(pos) + MixedArray::Elm::dataOff();
 711
 712   auto& v = vmain(env);
 713   loadTV(v, inst->dst(0), dstLoc(env, inst, 0), arr[off]);
 714 }
 715
 716 void cgArraySet(IRLS& env, const IRInstruction* i)    { implArraySet(env, i); }
 717 void cgArraySetRef(IRLS& env, const IRInstruction* i) { implArraySet(env, i); }
 718
 719 IMPL_OPCODE_CALL(SetNewElemArray);
 720
 721 IMPL_OPCODE_CALL(AddElemIntKey);
 722 IMPL_OPCODE_CALL(AddNewElem);
 723
 724 static ArrayData* addNewElemKeysetImpl(ArrayData* keyset, Cell v) {
 725   assertx(keyset->isKeyset());
 726   auto out = SetArray::Append(keyset, v, keyset->cowCheck());
 727   if (keyset != out) decRefArr(keyset);
 728   return out;
 729 }
 730
 731 static ArrayData* addNewElemVecImpl(ArrayData* vec, Cell v) {
 732   assertx(vec->isVecArray());
 733   auto out = PackedArray::AppendVec(vec, v, vec->cowCheck());
 734   if (vec != out) decRefArr(vec);
 735   return out;
 736 }
 737
 738 void cgAddNewElemKeyset(IRLS& env, const IRInstruction* inst) {
 739   cgCallHelper(
 740     vmain(env),
 741     env,
 742     CallSpec::direct(addNewElemKeysetImpl),
 743     callDest(env, inst),
 744     SyncOptions::Sync,
 745     argGroup(env, inst).ssa(0).typedValue(1)
 746   );
 747 }
 748
 749 void cgAddNewElemVec(IRLS& env, const IRInstruction* inst) {
 750   cgCallHelper(
 751     vmain(env),
 752     env,
 753     CallSpec::direct(addNewElemVecImpl),
 754     callDest(env, inst),
 755     SyncOptions::None,
 756     argGroup(env, inst).ssa(0).typedValue(1)
 757   );
 758 }
 759
 760 void cgAddElemStrKey(IRLS& env, const IRInstruction* inst) {
 761   auto& v = vmain(env);
 762
 763   auto const target = checkHACIntishCast()
 764     ? CallSpec::direct(addElemStringKeyHelper<true>)
 765     : CallSpec::direct(addElemStringKeyHelper<false>);
 766
 767   cgCallHelper(v, env, target,
 768                callDest(env, inst), SyncOptions::Sync,
 769                argGroup(env, inst).ssa(0).ssa(1).typedValue(2));
 770 }
 771
 772 void cgArrayIsset(IRLS& env, const IRInstruction* inst) {
 773   auto const arr     = inst->src(0);
 774   auto const key     = inst->src(1);
 775   auto const keyInfo = checkStrictlyInteger(arr->type(), key->type());
 776   BUILD_OPTAB(ARRAY_ISSET_HELPER_TABLE, keyInfo.type, keyInfo.checkForInt,
 777               checkHACIntishCast());
 778
 779   auto& v = vmain(env);
 780   cgCallHelper(v, env, CallSpec::direct(opFunc), callDest(env, inst),
 781                SyncOptions::Sync, arrArgs(env, inst, keyInfo));
 782 }
 783
 784 void cgArrayIdx(IRLS& env, const IRInstruction* inst) {
 785   auto const arr     = inst->src(0);
 786   auto const key     = inst->src(1);
 787   auto const keyInfo = checkStrictlyInteger(arr->type(), key->type());
 788
 789   auto const target = [&] () -> CallSpec {
 790     if (keyInfo.checkForInt) {
 791       return checkHACIntishCast()
 792         ? CallSpec::direct(arrayIdxSi<true>)
 793         : CallSpec::direct(arrayIdxSi<false>);
 794     }
 795     if (keyInfo.type == KeyType::Int) {
 796       return CallSpec::direct(arrayIdxI);
 797     }
 798     return CallSpec::direct(arrayIdxS);
 799   }();
 800
 801   auto& v = vmain(env);
 802   cgCallHelper(v, env, target, callDestTV(env, inst), SyncOptions::Sync,
 803                arrArgs(env, inst, keyInfo).typedValue(2));
 804 }
 805
 806 ///////////////////////////////////////////////////////////////////////////////
 807 // Vec.
 808
 809 namespace {
 810
 811 Vptr implPackedLayoutElemAddr(IRLS& env, Vloc arrLoc,
 812                               Vloc idxLoc, const SSATmp* idx) {
 813   auto const rarr = arrLoc.reg();
 814   auto const ridx = idxLoc.reg();
 815   auto& v = vmain(env);
 816
 817   static_assert(sizeof(TypedValue) == 16, "");
 818
 819   if (idx->hasConstVal()) {
 820     auto const offset = PackedArray::entriesOffset() +
 821                         idx->intVal() * sizeof(TypedValue);
 822     if (deltaFits(offset, sz::dword)) {
 823       return rarr[offset];
 824     }
 825   }
 826
 827   /*
 828    * Compute `rarr + ridx * sizeof(TypedValue) + PackedArray::entriesOffset()`.
 829    *
 830    * The logic of `scaledIdx * 16` is split in the following two instructions,
 831    * in order to save a byte in the shl instruction.
 832    *
 833    * TODO(#7728856): We should really move this into vasm-x64.cpp...
 834    */
 835   auto idxl = v.makeReg();
 836   auto scaled_idxl = v.makeReg();
 837   auto scaled_idx = v.makeReg();
 838   v << movtql{ridx, idxl};
 839   v << shlli{1, idxl, scaled_idxl, v.makeReg()};
 840   v << movzlq{scaled_idxl, scaled_idx};
 841   return rarr[scaled_idx * int(sizeof(TypedValue) / 2)
 842               + PackedArray::entriesOffset()];
 843 }
 844
 845 void implVecSet(IRLS& env, const IRInstruction* inst) {
 846   bool const setRef = inst->op() == VecSetRef;
 847   BUILD_OPTAB(VECSET_HELPER_TABLE, setRef);
 848
 849   auto args = argGroup(env, inst).
 850     ssa(0).
 851     ssa(1).
 852     typedValue(2);
 853   if (setRef) args.ssa(3);
 854
 855   auto& v = vmain(env);
 856   cgCallHelper(v, env, CallSpec::direct(opFunc), callDest(env, inst),
 857                SyncOptions::Sync, args);
 858 }
 859
 860 /*
 861  * Thread-local RDS packed array access sampling counter.
 862  */
 863 rds::Link<uint32_t, rds::Mode::Local> s_counter;
 864
 865 }
 866
 867 void record_packed_access(const ArrayData* ad) {
 868   assertx(s_counter.bound());
 869   *s_counter = RuntimeOption::EvalProfPackedArraySampleFreq;
 870
 871   auto record = StructuredLogEntry{};
 872   record.setInt("size", ad->size());
 873
 874   auto const st = StackTrace(StackTrace::Force{});
 875   auto frames = std::vector<folly::StringPiece>{};
 876   folly::split("\n", st.toString(), frames);
 877   record.setVec("stacktrace", frames);
 878
 879   FTRACE_MOD(Trace::prof_array, 1,
 880              "prof_array: {}\n", show(record).c_str());
 881   StructuredLog::log("hhvm_arrays", record);
 882 }
 883
 884 void cgLdPackedArrayDataElemAddr(IRLS& env, const IRInstruction* inst) {
 885   auto const arrLoc = srcLoc(env, inst, 0);
 886   auto const idxLoc = srcLoc(env, inst, 1);
 887   auto& v = vmain(env);
 888   auto& vc = vcold(env);
 889
 890   if (UNLIKELY(RuntimeOption::EvalProfPackedArraySampleFreq > 0)) {
 891     auto const arrTy = inst->src(0)->type();
 892     auto const packedTy = Type::Array(ArrayData::kPackedKind);
 893
 894     if (arrTy.maybe(packedTy)) {
 895       s_counter.bind(rds::Mode::Local);
 896
 897       auto const profile = [&] (Vout& v) {
 898         auto const handle = s_counter.handle();
 899         auto const sf = v.makeReg();
 900         v << declm{rvmtl()[handle], sf};
 901
 902         unlikelyIfThen(v, vc, CC_LE, sf, [&] (Vout& v) {
 903           // Log this array access.
 904           v << vcall{CallSpec::direct(record_packed_access),
 905                      v.makeVcallArgs({{arrLoc.reg()}}), v.makeTuple({})};
 906         });
 907       };
 908
 909       if (arrTy <= packedTy) {
 910         profile(v);
 911       } else {
 912         auto const sf = v.makeReg();
 913         v << cmpbim{ArrayData::kPackedKind, arrLoc.reg()[HeaderKindOffset], sf};
 914         ifThen(v, CC_E, sf, profile);
 915       }
 916     }
 917   }
 918
 919   auto const addr = implPackedLayoutElemAddr(env, arrLoc, idxLoc, inst->src(1));
 920   vmain(env) << lea{addr, dstLoc(env, inst, 0).reg()};
 921 }
 922
 923 namespace {
 924
 925 void packedLayoutLoadImpl(IRLS& env, const IRInstruction* inst) {
 926   auto const arrLoc = srcLoc(env, inst, 0);
 927   auto const idxLoc = srcLoc(env, inst, 1);
 928   auto const addr = implPackedLayoutElemAddr(env, arrLoc, idxLoc, inst->src(1));
 929   loadTV(vmain(env), inst->dst(), dstLoc(env, inst, 0), addr);
 930 }
 931
 932 }
 933
 934 void cgLdVecElem(IRLS& env, const IRInstruction* inst) {
 935   packedLayoutLoadImpl(env, inst);
 936 }
 937
 938 void cgLdPackedElem(IRLS& env, const IRInstruction* inst) {
 939   packedLayoutLoadImpl(env, inst);
 940 }
 941
 942 IMPL_OPCODE_CALL(ElemVecD)
 943 IMPL_OPCODE_CALL(ElemVecU)
 944
 945 void cgVecSet(IRLS& env, const IRInstruction* i)    { implVecSet(env, i); }
 946 void cgVecSetRef(IRLS& env, const IRInstruction* i) { implVecSet(env, i); }
 947
 948 IMPL_OPCODE_CALL(SetNewElemVec);
 949
 950 void cgReservePackedArrayDataNewElem(IRLS& env, const IRInstruction* i) {
 951   static_assert(ArrayData::sizeofSize() == 4, "");
 952
 953   auto& v = vmain(env);
 954   auto arrayData = srcLoc(env, i, 0).reg();
 955   auto const sizePtr = arrayData[ArrayData::offsetofSize()];
 956
 957   // If the check below succeeds, we'll end up returning the original size
 958   // so just use the destination register to hold the orignal size
 959   auto const size = dstLoc(env, i, 0).reg();
 960   v << loadzlq{sizePtr, size};
 961
 962   { // Bail out unless size < cap
 963     auto const indexb = v.makeReg();
 964     v << loadb{arrayData[PackedArray::SizeIndexOffset], indexb};
 965     auto const index = v.makeReg();
 966     v << movzbq{indexb, index};
 967     auto const cap = v.makeReg();
 968     auto const table =
 969       reinterpret_cast<uintptr_t>(kSizeIndex2PackedArrayCapacity);
 970     if (table < std::numeric_limits<int>::max()) {
 971       v << loadzlq{baseless(index * 4 + table), cap};
 972     } else {
 973       auto const base = v.cns(table);
 974       v << loadzlq{base[index * 4], cap};
 975     }
 976
 977     auto const sf = v.makeReg();
 978     v << cmpq{size, cap, sf};
 979     ifThen(v, CC_BE, sf, label(env, i->taken()));
 980   }
 981
 982   v << inclm{sizePtr, v.makeReg()};
 983 }
 984
 985 ///////////////////////////////////////////////////////////////////////////////
 986 // Dict.
 987
 988 namespace {
 989
 990 void implDictGet(IRLS& env, const IRInstruction* inst) {
 991   auto const key = inst->src(1);
 992   auto const mode =
 993     (inst->op() == DictGetQuiet) ? MOpMode::None : MOpMode::Warn;
 994   BUILD_OPTAB(DICTGET_HELPER_TABLE, getKeyType(key), mode);
 995
 996   auto args = argGroup(env, inst).ssa(0).ssa(1);
 997
 998   auto& v = vmain(env);
 999   cgCallHelper(v, env, CallSpec::direct(opFunc), callDestTV(env, inst),
1000                SyncOptions::Sync, args);
1001 }
1002
1003 void implDictSet(IRLS& env, const IRInstruction* inst) {
1004   auto const key = inst->src(1);
1005   bool const setRef  = inst->op() == DictSetRef;
1006   BUILD_OPTAB(DICTSET_HELPER_TABLE, getKeyType(key), setRef);
1007
1008   auto args = argGroup(env, inst).
1009     ssa(0).
1010     ssa(1).
1011     typedValue(2);
1012   if (setRef) args.ssa(3);
1013
1014   auto& v = vmain(env);
1015   cgCallHelper(v, env, CallSpec::direct(opFunc), callDest(env, inst),
1016                SyncOptions::Sync, args);
1017 }
1018
1019 void implDictIsset(IRLS& env, const IRInstruction* inst) {
1020   auto const key     = inst->src(1);
1021   auto const empty   = inst->op() == DictEmptyElem;
1022   BUILD_OPTAB(DICT_ISSET_EMPTY_ELEM_HELPER_TABLE, getKeyType(key), empty);
1023
1024   auto args = argGroup(env, inst).ssa(0).ssa(1);
1025
1026   auto& v = vmain(env);
1027   cgCallHelper(v, env, CallSpec::direct(opFunc), callDest(env, inst),
1028                SyncOptions::Sync, args);
1029 }
1030
1031 }
1032
1033 void cgElemDictX(IRLS& env, const IRInstruction* inst) {
1034   auto const key = inst->src(1);
1035   auto const mode = inst->extra<ElemDictX>()->mode;
1036   BUILD_OPTAB(ELEM_DICT_HELPER_TABLE, getKeyType(key), mode);
1037
1038   auto args = argGroup(env, inst).ssa(0).ssa(1);
1039
1040   auto& v = vmain(env);
1041   cgCallHelper(v, env, CallSpec::direct(opFunc), callDest(env, inst),
1042                SyncOptions::Sync, args);
1043 }
1044
1045 void cgElemDictD(IRLS& env, const IRInstruction* inst) {
1046   auto const key     = inst->src(1);
1047   BUILD_OPTAB(ELEM_DICT_D_HELPER_TABLE, getKeyType(key));
1048
1049   auto args = argGroup(env, inst).ssa(0).ssa(1);
1050
1051   auto& v = vmain(env);
1052   cgCallHelper(v, env, CallSpec::direct(opFunc), callDest(env, inst),
1053                SyncOptions::Sync, args);
1054 }
1055
1056 void cgElemDictU(IRLS& env, const IRInstruction* inst) {
1057   auto const key     = inst->src(1);
1058   BUILD_OPTAB(ELEM_DICT_U_HELPER_TABLE, getKeyType(key));
1059
1060   auto args = argGroup(env, inst).ssa(0).ssa(1);
1061
1062   auto& v = vmain(env);
1063   cgCallHelper(v, env, CallSpec::direct(opFunc), callDest(env, inst),
1064                SyncOptions::Sync, args);
1065 }
1066
1067 void cgElemDictK(IRLS& env, const IRInstruction* inst) {
1068   auto const dict = srcLoc(env, inst, 0).reg();
1069   auto const dst = dstLoc(env, inst, 0);
1070   auto const pos = inst->extra<ElemDictK>()->index;
1071   auto const off = MixedArray::elmOff(pos) + MixedArray::Elm::dataOff();
1072
1073   auto& v = vmain(env);
1074
1075   assertx(dst.numAllocated() == 1);
1076   v << lea{dict[off], dst.reg()};
1077 }
1078
1079 void cgDictGet(IRLS& env, const IRInstruction* inst) {
1080   implDictGet(env, inst);
1081 }
1082 void cgDictGetQuiet(IRLS& env, const IRInstruction* inst) {
1083   implDictGet(env, inst);
1084 }
1085
1086 void cgDictGetK(IRLS& env, const IRInstruction* inst) {
1087   auto const dict = srcLoc(env, inst, 0).reg();
1088   auto const pos = inst->extra<DictGetK>()->index;
1089   auto const off = MixedArray::elmOff(pos) + MixedArray::Elm::dataOff();
1090
1091   auto& v = vmain(env);
1092   loadTV(v, inst->dst(0), dstLoc(env, inst, 0), dict[off]);
1093 }
1094
1095 void cgDictSet(IRLS& env, const IRInstruction* i)    { implDictSet(env, i); }
1096 void cgDictSetRef(IRLS& env, const IRInstruction* i) { implDictSet(env, i); }
1097
1098 IMPL_OPCODE_CALL(DictAddElemIntKey);
1099 IMPL_OPCODE_CALL(DictAddElemStrKey);
1100
1101 void cgDictIsset(IRLS& env, const IRInstruction* inst) {
1102   implDictIsset(env, inst);
1103 }
1104 void cgDictEmptyElem(IRLS& env, const IRInstruction* inst) {
1105   implDictIsset(env, inst);
1106 }
1107
1108 void cgDictIdx(IRLS& env, const IRInstruction* inst) {
1109   auto const key     = inst->src(1);
1110   auto const target  = getKeyType(key) == KeyType::Int
1111     ? CallSpec::direct(dictIdxI)
1112     : CallSpec::direct(dictIdxS);
1113   auto args = argGroup(env, inst).ssa(0).ssa(1).typedValue(2);
1114   auto& v = vmain(env);
1115   cgCallHelper(v, env, target, callDestTV(env, inst),
1116                SyncOptions::Sync, args);
1117 }
1118
1119 ///////////////////////////////////////////////////////////////////////////////
1120 // Keyset.
1121
1122 namespace {
1123
1124 void implKeysetGet(IRLS& env, const IRInstruction* inst) {
1125   auto const key = inst->src(1);
1126   auto const mode = inst->op() == KeysetGetQuiet
1127     ? MOpMode::None
1128     : MOpMode::Warn;
1129   BUILD_OPTAB(KEYSETGET_HELPER_TABLE, getKeyType(key), mode);
1130
1131   auto args = argGroup(env, inst).ssa(0).ssa(1);
1132
1133   auto& v = vmain(env);
1134   cgCallHelper(v, env, CallSpec::direct(opFunc), callDestTV(env, inst),
1135                SyncOptions::Sync, args);
1136 }
1137
1138 void implKeysetIsset(IRLS& env, const IRInstruction* inst) {
1139   auto const key = inst->src(1);
1140   auto const empty = inst->op() == KeysetEmptyElem;
1141   BUILD_OPTAB(KEYSET_ISSET_EMPTY_ELEM_HELPER_TABLE, getKeyType(key), empty);
1142
1143   auto args = argGroup(env, inst).ssa(0).ssa(1);
1144
1145   auto& v = vmain(env);
1146   cgCallHelper(v, env, CallSpec::direct(opFunc), callDest(env, inst),
1147                SyncOptions::Sync, args);
1148 }
1149
1150 }
1151
1152 void cgElemKeysetX(IRLS& env, const IRInstruction* inst) {
1153   auto const key = inst->src(1);
1154   auto const mode = inst->extra<ElemKeysetX>()->mode;
1155   BUILD_OPTAB(ELEM_KEYSET_HELPER_TABLE, getKeyType(key), mode);
1156
1157   auto args = argGroup(env, inst).ssa(0).ssa(1);
1158
1159   auto& v = vmain(env);
1160   cgCallHelper(v, env, CallSpec::direct(opFunc), callDest(env, inst),
1161                SyncOptions::Sync, args);
1162 }
1163
1164 void cgElemKeysetU(IRLS& env, const IRInstruction* inst) {
1165   auto const key = inst->src(1);
1166   BUILD_OPTAB(ELEM_KEYSET_U_HELPER_TABLE, getKeyType(key));
1167
1168   auto args = argGroup(env, inst).ssa(0).ssa(1);
1169
1170   auto& v = vmain(env);
1171   cgCallHelper(v, env, CallSpec::direct(opFunc), callDest(env, inst),
1172                SyncOptions::Sync, args);
1173 }
1174
1175 void cgElemKeysetK(IRLS& env, const IRInstruction* inst) {
1176   auto const keyset = srcLoc(env, inst, 0).reg();
1177   auto const dst = dstLoc(env, inst, 0);
1178   auto const pos = inst->extra<ElemKeysetK>()->index;
1179   auto const off = SetArray::tvOff(pos);
1180
1181   auto& v = vmain(env);
1182   assertx(dst.numAllocated() == 1);
1183   v << lea{keyset[off], dst.reg()};
1184 }
1185
1186 void cgKeysetGet(IRLS& env, const IRInstruction* inst) {
1187   implKeysetGet(env, inst);
1188 }
1189 void cgKeysetGetQuiet(IRLS& env, const IRInstruction* inst) {
1190   implKeysetGet(env, inst);
1191 }
1192
1193 void cgKeysetGetK(IRLS& env, const IRInstruction* inst) {
1194   auto const keyset = srcLoc(env, inst, 0).reg();
1195   auto const pos = inst->extra<KeysetGetK>()->index;
1196   auto const off = SetArray::tvOff(pos);
1197
1198   auto& v = vmain(env);
1199   loadTV(v, inst->dst(0), dstLoc(env, inst, 0), keyset[off]);
1200 }
1201
1202 void cgSetNewElemKeyset(IRLS& env, const IRInstruction* inst) {
1203   auto const key     = inst->src(1);
1204   BUILD_OPTAB(KEYSET_SETNEWELEM_HELPER_TABLE, getKeyType(key));
1205
1206   auto args = argGroup(env, inst).ssa(0).ssa(1);
1207
1208   auto& v = vmain(env);
1209   cgCallHelper(v, env, CallSpec::direct(opFunc), callDest(env, inst),
1210                SyncOptions::Sync, args);
1211 }
1212
1213 void cgKeysetIsset(IRLS& env, const IRInstruction* inst) {
1214   implKeysetIsset(env, inst);
1215 }
1216 void cgKeysetEmptyElem(IRLS& env, const IRInstruction* inst) {
1217   implKeysetIsset(env, inst);
1218 }
1219
1220 void cgKeysetIdx(IRLS& env, const IRInstruction* inst) {
1221   auto const key     = inst->src(1);
1222   auto const target  = getKeyType(key) == KeyType::Int
1223     ? CallSpec::direct(keysetIdxI)
1224     : CallSpec::direct(keysetIdxS);
1225   auto args = argGroup(env, inst).ssa(0).ssa(1).typedValue(2);
1226   auto& v = vmain(env);
1227   cgCallHelper(v, env, target, callDestTV(env, inst),
1228                SyncOptions::Sync, args);
1229 }
1230
1231 ///////////////////////////////////////////////////////////////////////////////
1232 // Collections.
1233
1234 IMPL_OPCODE_CALL(PairIsset);
1235 IMPL_OPCODE_CALL(VectorIsset);
1236
1237 void cgMapGet(IRLS& env, const IRInstruction* inst) {
1238   auto const target = inst->src(1)->isA(TInt)
1239     ? CallSpec::direct(MInstrHelpers::mapGetImpl<KeyType::Int>)
1240     : CallSpec::direct(MInstrHelpers::mapGetImpl<KeyType::Str>);
1241
1242   auto const args = argGroup(env, inst).ssa(0).ssa(1);
1243
1244   auto& v = vmain(env);
1245   cgCallHelper(v, env, target, callDestTV(env, inst),
1246                SyncOptions::Sync, args);
1247 }
1248
1249 void cgMapSet(IRLS& env, const IRInstruction* inst) {
1250   auto const target = inst->src(1)->isA(TInt)
1251     ? CallSpec::direct(MInstrHelpers::mapSetImpl<KeyType::Int>)
1252     : CallSpec::direct(MInstrHelpers::mapSetImpl<KeyType::Str>);
1253
1254   auto const args = argGroup(env, inst)
1255     .ssa(0)
1256     .ssa(1)
1257     .typedValue(2);
1258
1259   auto& v = vmain(env);
1260   cgCallHelper(v, env, target, kVoidDest, SyncOptions::Sync, args);
1261 }
1262
1263 void cgMapIsset(IRLS& env, const IRInstruction* inst) {
1264   auto const target = inst->src(1)->isA(TInt)
1265     ? CallSpec::direct(MInstrHelpers::mapIssetImpl<KeyType::Int>)
1266     : CallSpec::direct(MInstrHelpers::mapIssetImpl<KeyType::Str>);
1267
1268   auto const args = argGroup(env, inst).ssa(0).ssa(1);
1269
1270   auto& v = vmain(env);
1271   cgCallHelper(v, env, target, callDest(env, inst),
1272                SyncOptions::Sync, args);
1273 }
1274
1275 ///////////////////////////////////////////////////////////////////////////////
1276
1277 void cgMemoGet(IRLS& env, const IRInstruction* inst) {
1278   auto const extra = inst->extra<MemoGet>();
1279   auto const fp    = srcLoc(env, inst, 0).reg();
1280   auto& v          = vmain(env);
1281
1282   auto const args = argGroup(env, inst)
1283     .ssa(1)
1284     .addr(fp, localOffset(extra->locals.first))
1285     .imm(extra->locals.restCount + 1);
1286   cgCallHelper(
1287     v, env, CallSpec::direct(MixedArray::MemoGet),
1288     callDestTV(env, inst), SyncOptions::None, args
1289   );
1290 }
1291
1292 void cgMemoSet(IRLS& env, const IRInstruction* inst) {
1293   auto const extra = inst->extra<MemoSet>();
1294   auto const fp    = srcLoc(env, inst, 0).reg();
1295   auto& v          = vmain(env);
1296
1297   auto const args = argGroup(env, inst)
1298     .ssa(1)
1299     .addr(fp, localOffset(extra->locals.first))
1300     .imm(extra->locals.restCount + 1)
1301     .typedValue(2);
1302   cgCallHelper(
1303     v, env, CallSpec::direct(MixedArray::MemoSet),
1304     kVoidDest, SyncOptions::Sync, args
1305   );
1306 }
1307
1308 ///////////////////////////////////////////////////////////////////////////////
1309
1310 }}}