hphp/runtime/vm/jit/irlower-minstr.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/runtime/vm/jit/irlower-internal.h"
  18
  19 #include "hphp/runtime/base/array-data.h"
  20 #include "hphp/runtime/base/header-kind.h"
  21 #include "hphp/runtime/base/rds.h"
  22 #include "hphp/runtime/base/str-key-table.h"
  23 #include "hphp/runtime/base/typed-value.h"
  24 #include "hphp/runtime/base/unaligned-typed-value.h"
  25 #include "hphp/runtime/base/vanilla-dict.h"
  26 #include "hphp/runtime/base/vanilla-vec-defs.h"
  27 #include "hphp/runtime/base/vanilla-vec.h"
  28 #include "hphp/runtime/vm/member-operations.h"
  29 #include "hphp/runtime/vm/property-profile.h"
  30 #include "hphp/runtime/vm/unit.h"
  31
  32 #include "hphp/runtime/vm/jit/abi.h"
  33 #include "hphp/runtime/vm/jit/arg-group.h"
  34 #include "hphp/runtime/vm/jit/array-access-profile.h"
  35 #include "hphp/runtime/vm/jit/bc-marker.h"
  36 #include "hphp/runtime/vm/jit/code-gen-cf.h"
  37 #include "hphp/runtime/vm/jit/code-gen-helpers.h"
  38 #include "hphp/runtime/vm/jit/cow-profile.h"
  39 #include "hphp/runtime/vm/jit/decref-profile.h"
  40 #include "hphp/runtime/vm/jit/ir-instruction.h"
  41 #include "hphp/runtime/vm/jit/irlower-bespoke.h"
  42 #include "hphp/runtime/vm/jit/minstr-helpers.h"
  43 #include "hphp/runtime/vm/jit/translator-inline.h"
  44 #include "hphp/runtime/vm/jit/translator-runtime.h"
  45
  46 #include "hphp/util/immed.h"
  47 #include "hphp/util/stack-trace.h"
  48 #include "hphp/util/struct-log.h"
  49 #include "hphp/util/trace.h"
  50
  51 // This file does ugly things with macros so include last.
  52 #include "hphp/runtime/vm/jit/irlower-minstr-internal.h"
  53
  54 namespace HPHP::jit::irlower {
  55
  56 TRACE_SET_MOD(irlower);
  57
  58 ///////////////////////////////////////////////////////////////////////////////
  59
  60 void cgBaseG(IRLS& env, const IRInstruction* inst) {
  61   auto const mode = inst->extra<MOpModeData>()->mode;
  62   BUILD_OPTAB(BASE_G_HELPER_TABLE, mode);
  63
  64   auto const args = argGroup(env, inst).typedValue(0);
  65
  66   auto& v = vmain(env);
  67   cgCallHelper(v, env, target, callDest(env, inst), SyncOptions::Sync, args);
  68 }
  69
  70 void cgFinishMemberOp(IRLS&, const IRInstruction*) {}
  71
  72 ///////////////////////////////////////////////////////////////////////////////
  73
  74 namespace {
  75
  76 /*
  77  * Make an ArgGroup for prop instructions that takes:
  78  *    1/ the context Class*
  79  *    2/ the ObjectData* or a TypedValue, depending on the base type
  80  */
  81 ArgGroup propArgs(IRLS& env, const IRInstruction* inst) {
  82   auto const base = inst->src(0);
  83   auto args = argGroup(env, inst).immPtr(inst->marker().func());
  84   if (base->isA(TObj)) return args.ssa(0);
  85   return args.typedValue(0);
  86 }
  87
  88 void implProp(IRLS& env, const IRInstruction* inst) {
  89   auto const mode    = inst->extra<PropData>()->mode;
  90   auto const base    = inst->src(0);
  91   auto const key     = inst->src(1);
  92   auto const keyType = getKeyTypeNoInt(key);
  93
  94   auto const args = propArgs(env, inst)
  95     .memberKeyS(1)
  96     .ssa(2)
  97     .imm(static_cast<int32_t>(inst->extra<PropData>()->op));
  98
  99   auto const target = [&] {
 100     if (inst->is(PropDX)) {
 101       BUILD_OPTAB2(base->isA(TObj),
 102                    PROPD_OBJ_HELPER_TABLE,
 103                    PROPD_HELPER_TABLE,
 104                    keyType);
 105       return target;
 106     } else {
 107       BUILD_OPTAB2(base->isA(TObj),
 108                    PROP_OBJ_HELPER_TABLE,
 109                    PROP_HELPER_TABLE,
 110                    mode, keyType);
 111       return target;
 112     }
 113   }();
 114
 115   auto& v = vmain(env);
 116   cgCallHelper(v, env, target, callDest(env, inst), SyncOptions::Sync, args);
 117 }
 118
 119 }
 120
 121 ///////////////////////////////////////////////////////////////////////////////
 122
 123 void cgPropX(IRLS& env, const IRInstruction* i)  { implProp(env, i); }
 124 void cgPropDX(IRLS& env, const IRInstruction* i) { implProp(env, i); }
 125
 126 void cgPropQ(IRLS& env, const IRInstruction* inst) {
 127   using namespace MInstrHelpers;
 128
 129   auto const args = propArgs(env, inst)
 130     .ssa(1)
 131     .ssa(2)
 132     .imm(static_cast<int32_t>(inst->extra<PropData>()->op));
 133
 134   auto& v = vmain(env);
 135   if (inst->src(0)->isA(TObj)) {
 136     return cgCallHelper(v, env, CallSpec::direct(propCOQ), callDest(env, inst),
 137                         SyncOptions::Sync, args);
 138   }
 139
 140   auto const mode = inst->extra<PropData>()->mode;
 141   BUILD_OPTAB(PROPQ_HELPER_TABLE, mode);
 142   cgCallHelper(v, env, target, callDest(env, inst), SyncOptions::Sync, args);
 143 }
 144
 145 void cgCGetProp(IRLS& env, const IRInstruction* inst) {
 146   auto const mode    = inst->extra<PropData>()->mode;
 147   auto const base    = inst->src(0);
 148   auto const key     = inst->src(1);
 149   auto const keyType = getKeyTypeNoInt(key);
 150
 151   BUILD_OPTAB2(base->isA(TObj),
 152                CGET_OBJ_PROP_HELPER_TABLE,
 153                CGET_PROP_HELPER_TABLE,
 154                keyType, mode);
 155
 156   auto const args = propArgs(env, inst)
 157     .memberKeyS(1)
 158     .imm(static_cast<int32_t>(inst->extra<PropData>()->op));
 159
 160   auto& v = vmain(env);
 161   cgCallHelper(v, env, target, callDestTV(env, inst), SyncOptions::Sync, args);
 162 }
 163
 164 void cgCGetPropQ(IRLS& env, const IRInstruction* inst) {
 165   using namespace MInstrHelpers;
 166   auto const mode = inst->extra<PropData>()->mode;
 167
 168   auto args = propArgs(env, inst)
 169     .ssa(1)
 170     .imm(static_cast<int32_t>(inst->extra<PropData>()->op));
 171
 172   auto& v = vmain(env);
 173
 174   if (inst->src(0)->isA(TObj)) {
 175     return cgCallHelper(v, env, CallSpec::direct(cGetPropSOQ),
 176                         callDestTV(env, inst), SyncOptions::Sync, args);
 177   }
 178
 179   BUILD_OPTAB(CGET_PROPQ_HELPER_TABLE, mode);
 180   cgCallHelper(v, env, target, callDestTV(env, inst), SyncOptions::Sync, args);
 181 }
 182
 183 void cgSetProp(IRLS& env, const IRInstruction* inst) {
 184   auto const base = inst->src(0);
 185   auto const key = inst->src(1);
 186   auto const keyType = getKeyTypeNoInt(key);
 187
 188   auto args = propArgs(env, inst)
 189     .memberKeyS(1)
 190     .typedValue(2)
 191     .imm(static_cast<int32_t>(inst->extra<ReadonlyData>()->op));
 192
 193   auto const target = [&] {
 194     if (base->isA(TObj)) {
 195       BUILD_OPTAB(SETPROP_OBJ_HELPER_TABLE, keyType);
 196       return target;
 197     } else {
 198       BUILD_OPTAB(SETPROP_HELPER_TABLE, keyType);
 199       return target;
 200     }
 201   }();
 202
 203   auto& v = vmain(env);
 204   cgCallHelper(v, env, target, kVoidDest, SyncOptions::Sync, args);
 205 }
 206
 207 void cgUnsetProp(IRLS& env, const IRInstruction* inst) {
 208   auto const base = inst->src(0);
 209
 210   auto helper = base->isA(TObj)
 211     ? CallSpec::direct(MInstrHelpers::unsetPropCO)
 212     : CallSpec::direct(MInstrHelpers::unsetPropC);
 213
 214   auto const args = propArgs(env, inst).typedValue(1);
 215
 216   auto& v = vmain(env);
 217   cgCallHelper(v, env, helper, kVoidDest, SyncOptions::Sync, args);
 218 }
 219
 220 void cgSetOpProp(IRLS& env, const IRInstruction* inst) {
 221   auto const base = inst->src(0);
 222   auto const extra = inst->extra<SetOpProp>();
 223
 224   auto helper = base->isA(TObj)
 225     ? CallSpec::direct(MInstrHelpers::setOpPropCO)
 226     : CallSpec::direct(MInstrHelpers::setOpPropC);
 227
 228   auto args = propArgs(env, inst)
 229     .typedValue(1)
 230     .typedValue(2)
 231     .imm(static_cast<int32_t>(extra->op));
 232
 233   auto& v = vmain(env);
 234   cgCallHelper(v, env, helper, callDestTV(env, inst), SyncOptions::Sync, args);
 235 }
 236
 237 void cgIncDecProp(IRLS& env, const IRInstruction* inst) {
 238   auto const base = inst->src(0);
 239   auto const extra = inst->extra<IncDecProp>();
 240
 241   auto helper = base->isA(TObj)
 242     ? CallSpec::direct(MInstrHelpers::incDecPropCO)
 243     : CallSpec::direct(MInstrHelpers::incDecPropC);
 244
 245   auto args = propArgs(env, inst)
 246     .typedValue(1)
 247     .imm(static_cast<int32_t>(extra->op));
 248
 249   auto& v = vmain(env);
 250   cgCallHelper(v, env, helper, callDestTV(env, inst),
 251                SyncOptions::Sync, args);
 252 }
 253
 254 void cgIssetProp(IRLS& env, const IRInstruction* inst) {
 255   auto const base = inst->src(0);
 256   auto const key = inst->src(1);
 257   auto const keyType = getKeyTypeNoInt(key);
 258
 259   BUILD_OPTAB2(base->isA(TObj),
 260                ISSET_OBJ_PROP_HELPER_TABLE,
 261                ISSET_PROP_HELPER_TABLE,
 262                keyType);
 263
 264   auto const args = propArgs(env, inst).memberKeyS(1);
 265
 266   auto& v = vmain(env);
 267   cgCallHelper(v, env, target, callDest(env, inst), SyncOptions::Sync, args);
 268 }
 269
 270 void cgProfileProp(IRLS& env, const IRInstruction* inst) {
 271   auto const cls = inst->src(0)->strVal();
 272   auto const prop = inst->src(1)->strVal();
 273   auto const counterPtr = PropertyProfile::getCounterAddr(cls, prop);
 274   auto& v = vmain(env);
 275   auto const addr = v.makeReg();
 276   v << copy{v.cns(counterPtr), addr};
 277   v << inclm{addr[0], v.makeReg()};
 278 }
 279
 280
 281 ///////////////////////////////////////////////////////////////////////////////
 282
 283 namespace {
 284
 285 /*
 286  * Make an ArgGroup for elem instructions that takes:
 287  *    1/ the pointer to an array-like TV
 288  *    2/ the index key, as a raw value or a TypedValue depending on whether
 289  *       the type is known
 290  */
 291 ArgGroup elemArgs(IRLS& env, const IRInstruction* inst) {
 292   auto args = argGroup(env, inst);
 293   if (inst->src(0)->isA(TMem)) {
 294     args.ssa(0);
 295   } else {
 296     args.typedValue(0);
 297   }
 298   args.memberKeyIS(1);
 299   return args;
 300 }
 301
 302 void implElem(IRLS& env, const IRInstruction* inst) {
 303   auto const mode = inst->extra<MOpModeData>()->mode;
 304   auto const key  = inst->src(1);
 305   auto const sync = SyncOptions::Sync;
 306   auto args = elemArgs(env, inst);
 307
 308   if (inst->is(ElemDX)) {
 309     assertx(mode == MOpMode::Define);
 310     BUILD_OPTAB(ELEMD_HELPER_TABLE, getKeyType(key));
 311     cgCallHelper(vmain(env), env, target, callDest(env, inst), sync, args);
 312     return;
 313   }
 314
 315   if (inst->is(ElemUX)) {
 316     assertx(mode == MOpMode::Unset);
 317     BUILD_OPTAB(ELEMU_HELPER_TABLE, getKeyType(key));
 318     cgCallHelper(vmain(env), env, target, callDest(env, inst), sync, args);
 319     return;
 320   }
 321
 322   BUILD_OPTAB(ELEM_HELPER_TABLE, getKeyType(key), mode);
 323   cgCallHelper(vmain(env), env, target, callDestTV(env, inst), sync, args);
 324 }
 325
 326 }
 327
 328 ///////////////////////////////////////////////////////////////////////////////
 329
 330 void cgElemX(IRLS& env, const IRInstruction* i)  { implElem(env, i); }
 331 void cgElemDX(IRLS& env, const IRInstruction* i) { implElem(env, i); }
 332 void cgElemUX(IRLS& env, const IRInstruction* i) { implElem(env, i); }
 333
 334 void cgCGetElem(IRLS& env, const IRInstruction* inst) {
 335   auto const mode  = inst->extra<MOpModeData>()->mode;
 336   auto const key   = inst->src(1);
 337   BUILD_OPTAB(CGETELEM_HELPER_TABLE, getKeyType(key), mode);
 338
 339   auto& v = vmain(env);
 340   cgCallHelper(v, env, target, callDestTV(env, inst),
 341                SyncOptions::Sync, elemArgs(env, inst));
 342 }
 343
 344 void cgSetElem(IRLS& env, const IRInstruction* inst) {
 345   BUILD_OPTAB(SETELEM_HELPER_TABLE, getKeyType(inst->src(1)));
 346
 347   auto& v = vmain(env);
 348   cgCallHelper(v, env, target, callDest(env, inst),
 349                SyncOptions::Sync, elemArgs(env, inst).typedValue(2));
 350 }
 351
 352 void cgSetRange(IRLS& env, const IRInstruction* inst) {
 353   auto const target = inst->is(SetRangeRev) ?
 354     CallSpec::direct(HPHP::SetRange<true>) :
 355     CallSpec::direct(HPHP::SetRange<false>);
 356   cgCallHelper(
 357     vmain(env), env, target, callDest(env, inst),
 358     SyncOptions::Sync,
 359     argGroup(env, inst).ssa(0).ssa(1).typedValue(2).ssa(3).ssa(4)
 360   );
 361 }
 362
 363 void cgSetRangeRev(IRLS& env, const IRInstruction* inst) {
 364   cgSetRange(env, inst);
 365 }
 366
 367 void cgSetOpElem(IRLS& env, const IRInstruction* inst) {
 368   auto& v = vmain(env);
 369
 370   auto const target = CallSpec::direct(MInstrHelpers::setOpElem);
 371
 372   auto const args = argGroup(env, inst)
 373     .ssa(0)
 374     .typedValue(1)
 375     .typedValue(2)
 376     .imm(uint32_t(inst->extra<SetOpElem>()->op));
 377
 378   cgCallHelper(v, env, target, callDestTV(env, inst),
 379                SyncOptions::Sync, args);
 380 }
 381
 382 void cgIncDecElem(IRLS& env, const IRInstruction* inst) {
 383   auto& v = vmain(env);
 384
 385   auto const target = CallSpec::direct(MInstrHelpers::incDecElem);
 386
 387   auto const args = argGroup(env, inst)
 388     .ssa(0)
 389     .typedValue(1)
 390     .imm(uint32_t(inst->extra<IncDecElem>()->op));
 391
 392   cgCallHelper(v, env, target, callDestTV(env, inst),
 393                SyncOptions::Sync, args);
 394 }
 395
 396 void cgUnsetElem(IRLS& env, const IRInstruction* inst) {
 397   auto const key = inst->src(1);
 398   BUILD_OPTAB(UNSET_ELEM_HELPER_TABLE, getKeyType(key));
 399
 400   auto& v = vmain(env);
 401   cgCallHelper(v, env, target, kVoidDest,
 402                SyncOptions::Sync, elemArgs(env, inst));
 403 }
 404
 405 void cgIssetElem(IRLS& env, const IRInstruction* inst) {
 406   auto const key     = inst->src(1);
 407   BUILD_OPTAB(ISSET_ELEM_HELPER_TABLE, getKeyType(key));
 408
 409   auto& v = vmain(env);
 410   cgCallHelper(v, env, target, callDest(env, inst),
 411                SyncOptions::Sync, elemArgs(env, inst));
 412 }
 413
 414 ///////////////////////////////////////////////////////////////////////////////
 415 // Offset profiling.
 416
 417 namespace {
 418
 419 void implProfileHackArrayAccess(IRLS& env, const IRInstruction* inst,
 420                                 const CallSpec& target) {
 421   auto& v = vmain(env);
 422
 423   auto const extra = inst->extra<RDSHandlePairData>();
 424   auto args = argGroup(env, inst)
 425                 .ssa(0)
 426                 .ssa(1)
 427                 .addr(rvmtl(), safe_cast<int32_t>(extra->handle));
 428   if (extra->extra == rds::kUninitHandle) {
 429     args.immPtr(nullptr);
 430   } else {
 431     args.addr(rvmtl(), safe_cast<int32_t>(extra->extra));
 432   }
 433   cgCallHelper(v, env, target, kVoidDest, SyncOptions::Sync, args);
 434 }
 435
 436 }
 437
 438 void cgProfileDictAccess(IRLS& env, const IRInstruction* inst) {
 439   BUILD_OPTAB(PROFILE_DICT_ACCESS_HELPER_TABLE, getKeyType(inst->src(1)));
 440   implProfileHackArrayAccess(env, inst, target);
 441 }
 442
 443 void cgProfileKeysetAccess(IRLS& env, const IRInstruction* inst) {
 444   BUILD_OPTAB(PROFILE_KEYSET_ACCESS_HELPER_TABLE, getKeyType(inst->src(1)));
 445   implProfileHackArrayAccess(env, inst, target);
 446 }
 447
 448 void cgCheckDictOffset(IRLS& env, const IRInstruction* inst) {
 449   auto const arr = srcLoc(env, inst, 0).reg();
 450   auto const key = srcLoc(env, inst, 1).reg();
 451   auto const branch = label(env, inst->taken());
 452   auto const pos = inst->extra<IndexData>()->index;
 453   auto& v = vmain(env);
 454
 455   auto const elmOff = VanillaDict::elmOff(pos);
 456   using Elm = VanillaDict::Elm;
 457
 458   auto const key_type = getKeyType(inst->src(1));
 459   auto const is_str_key = key_type == KeyType::Str;
 460   assertx(key_type != KeyType::Any);
 461
 462   { // Also fail if our predicted position exceeds bounds. The layout-agnostic
 463     // variant does a test on the (m_size, m_extra) quadword here; for vanilla
 464     // array-likes, m_extra is always 0, and for bespoke array-likes, the full
 465     // quadword is always a negative int64_t.
 466     auto const sf = v.makeReg();
 467     v << cmplim{safe_cast<int32_t>(pos), arr[VanillaDict::usedOff()], sf};
 468     ifThen(v, CC_LE, sf, branch);
 469   }
 470   { // Fail if the Elm key value doesn't match.
 471     auto const sf = v.makeReg();
 472     v << cmpqm{key, arr[elmOff + Elm::keyOff()], sf};
 473     ifThen(v, CC_NE, sf, branch);
 474   }
 475   { // Fail if the Elm key type doesn't match.
 476     auto const sf = v.makeReg();
 477     v << cmplim{0, arr[elmOff + Elm::dataOff() + TVOFF(m_aux)], sf};
 478
 479     ifThen(v, is_str_key ? CC_L : CC_GE, sf, branch);
 480   }
 481   { // Fail if the Elm is a tombstone.  See VanillaDict::isTombstone().
 482     // We set the key type to look like an int when setting it be a tombstone.
 483     // This is originally to simplify the type scan, however we can use this
 484     // to elide this check when looking for string keys.
 485     if (!is_str_key) {
 486       auto const sf = v.makeReg();
 487       v << cmpbim{static_cast<data_type_t>(kInvalidDataType),
 488                   arr[elmOff + Elm::dataOff() + TVOFF(m_type)], sf};
 489       ifThen(v, CC_E, sf, branch);
 490     }
 491   }
 492 }
 493
 494 void cgCheckKeysetOffset(IRLS& env, const IRInstruction* inst) {
 495   auto const keyset = srcLoc(env, inst, 0).reg();
 496   auto const key = srcLoc(env, inst, 1).reg();
 497   auto const branch = label(env, inst->taken());
 498   auto const pos = inst->extra<CheckKeysetOffset>()->index;
 499   auto& v = vmain(env);
 500   auto const tvOff = pos * (int) sizeof(VanillaKeyset::Elm) +
 501                      VanillaKeyset::dataOff() + VanillaKeyset::tvOff();
 502
 503   { // Fail if our predicted position exceeds bounds.
 504     auto const sf = v.makeReg();
 505     v << cmplim{safe_cast<int32_t>(pos), keyset[VanillaKeyset::usedOff()], sf};
 506     ifThen(v, CC_LE, sf, branch);
 507   }
 508   { // Fail if the Elm key value doesn't match.
 509     auto const sf = v.makeReg();
 510     v << cmpqm{key, keyset[tvOff + TVOFF(m_data)], sf};
 511     ifThen(v, CC_NE, sf, branch);
 512   }
 513   { // Fail if the Elm key type doesn't match.
 514     auto const sf = v.makeReg();
 515     v << cmplim{0, keyset[tvOff + TVOFF(m_aux)], sf};
 516
 517     auto const key_type = getKeyType(inst->src(1));
 518     assertx(key_type != KeyType::Any);
 519
 520     auto const is_str_key = key_type == KeyType::Str;
 521     ifThen(v, is_str_key ? CC_L : CC_GE, sf, branch);
 522   }
 523   { // Fail if the Elm is a tombstone.  See VanillaKeyset::isTombstone().
 524     auto const sf = v.makeReg();
 525     v << cmpbim{static_cast<data_type_t>(kInvalidDataType),
 526                 keyset[tvOff + TVOFF(m_type)], sf};
 527     ifThen(v, CC_E, sf, branch);
 528   }
 529 }
 530
 531 void cgDictIterEnd(IRLS& env, const IRInstruction* inst) {
 532   static_assert(VanillaDict::usedSize() == 4);
 533   auto const dict = srcLoc(env, inst, 0).reg();
 534   auto const dst = dstLoc(env, inst, 0).reg();
 535   vmain(env) << loadzlq{dict[VanillaDict::usedOff()], dst};
 536 }
 537
 538 void cgKeysetIterEnd(IRLS& env, const IRInstruction* inst) {
 539   static_assert(VanillaKeyset::usedSize() == 4);
 540   auto const keyset = srcLoc(env, inst, 0).reg();
 541   auto const dst = dstLoc(env, inst, 0).reg();
 542   vmain(env) << loadzlq{keyset[VanillaKeyset::usedOff()], dst};
 543 }
 544
 545 void cgCheckDictKeys(IRLS& env, const IRInstruction* inst) {
 546   auto const src = srcLoc(env, inst, 0).reg();
 547   auto const mask = inst->extra<CheckDictKeys>()->keyTypes.toMissingBits();
 548   assertx(mask != 0);  // Do not emit pointless checks.
 549
 550   auto& v = vmain(env);
 551   auto const sf = v.makeReg();
 552   v << testbim{int8_t(mask), src[VanillaDict::kKeyTypesOffset], sf};
 553   v << jcc{CC_NZ, sf, {label(env, inst->next()), label(env, inst->taken())}};
 554 }
 555
 556 namespace {
 557 void traceCheckNotInStrKeyTable(ArrayData* ad, StringData* sd) {
 558   FTRACE_MOD(Trace::idx, 1,
 559              "Key: {}, HashMasked: {}, ad: {}\n",
 560              sd->data(),
 561              sd->hash() & StrKeyTable::kStrKeyTableMask,
 562              ad);
 563   FTRACE_MOD(Trace::idx, 1,
 564              "HasStrKeyTable: {}\nMay be in table: {}\n",
 565              ad->hasStrKeyTable(),
 566              ad->missingKeySideTable().mayContain(sd));
 567 }
 568 } // namespace
 569
 570 void cgCheckMissingKeyInArrLike(IRLS& env, const IRInstruction* inst) {
 571   auto const arr = srcLoc(env, inst, 0).reg();
 572   auto const key = srcLoc(env, inst, 1).reg();
 573   auto const branch = label(env, inst->taken());
 574   auto& v = vmain(env);
 575
 576   if (Trace::moduleEnabled(Trace::idx, 1)) {
 577     v << copy2{arr, srcLoc(env, inst, 1).reg(), rarg(0), rarg(1)};
 578     v << call{TCA(traceCheckNotInStrKeyTable), arg_regs(2)};
 579   }
 580
 581   // If the array doesn't have the table, jump to branch.
 582   auto const sf = v.makeReg();
 583   v << testbim{ArrayData::kHasStrKeyTable, arr[HeaderAuxOffset], sf};
 584   ifThen(v, CC_Z, sf, branch);
 585
 586   auto const mask = StrKeyTable::kStrKeyTableMask;
 587   auto const tableSize = safe_cast<int32_t>(sizeof(StrKeyTable));
 588   static_assert(sizeof(StrKeyTable) % 8 == 0,
 589                 "Size of StrKeyTable to be a multiple of 8 bytes");
 590   if (inst->src(1)->hasConstVal(TStaticStr)) {
 591     // If the key is constant, so is the hash.
 592     // So, emit a direct 1-byte test for the corresponding bit.
 593     auto const sfmap = v.makeReg();
 594     auto const maskedHash = inst->src(1)->strVal()->hash() & mask;
 595     v << testqim{
 596       1 << (maskedHash & 7),
 597       arr[(maskedHash >> 3) - tableSize],
 598       sfmap
 599     };
 600     ifThen(v, CC_NZ, sfmap, branch);
 601     return;
 602   }
 603   auto const hash = v.makeReg();       // sd->hash
 604   auto const maskedHash = v.makeReg(); // hash & mask
 605   auto const test = v.makeReg();       // 1 << maskedHash
 606   auto const sfmap = v.makeReg();
 607   // Mask the hash to get an index into the StrKeyTable.
 608   v << load{key[StringData::hashOff()], hash};
 609   v << andqi{mask, hash, maskedHash, v.makeReg()};
 610   auto const indexBits = [&] {
 611     if (tableSize == 8 /* bytes */) return maskedHash;
 612     auto const reg = v.makeReg();
 613     // Extract the bottom 6 bits to index into the quadword
 614     v << andqi{0x3F, maskedHash, reg, v.makeReg()};
 615     return reg;
 616   }();
 617   v << shl{indexBits, v.cns(1), test, v.makeReg()};
 618
 619   if (tableSize == 8 /* bytes */) {
 620     // If the bitset only has 64 bits, we can use a single quadword test.
 621     v << testqm{test, arr[-tableSize], sfmap};
 622   } else {
 623     // Otherwise, find the right quadword and apply the test.
 624     auto const offset = v.makeReg(); // maskedHash >> 6
 625     v << shrqi{6, maskedHash, offset, v.makeReg()};
 626     v << testqm{test, arr[offset * 8 - tableSize], sfmap};
 627   }
 628   // If the bit is set(i.e. the key may exist in the array), jump to the branch.
 629   ifThen(v, CC_NZ, sfmap, branch);
 630 }
 631
 632 ///////////////////////////////////////////////////////////////////////////////
 633 // COW
 634
 635 void cgCheckArrayCOW(IRLS& env, const IRInstruction* inst) {
 636   auto const arr = srcLoc(env, inst, 0).reg();
 637   auto const dst = dstLoc(env, inst, 0).reg();
 638   auto& v = vmain(env);
 639
 640   auto const sf = emitCmpRefCount(v, OneReference, arr);
 641   ifThen(v, CC_NE, sf, label(env, inst->taken()));
 642   v << copy{arr, dst};
 643 }
 644
 645 void cgCopyArray(IRLS& env, const IRInstruction* inst) {
 646   auto& v = vmain(env);
 647   auto const copy = copyFuncForArrayLike(inst->src(0)->type());
 648   auto const args = argGroup(env, inst).ssa(0);
 649   cgCallHelper(v, env, copy, callDest(env, inst), SyncOptions::None, args);
 650 }
 651
 652 void cgProfileArrayCOW(IRLS& env, const IRInstruction* inst) {
 653   auto& v = vmain(env);
 654   auto const extra = inst->extra<RDSHandleData>();
 655   auto const args =
 656     argGroup(env, inst).addr(rvmtl(), safe_cast<int32_t>(extra->handle)).ssa(0);
 657   cgCallHelper(
 658     v,
 659     env,
 660     CallSpec::method(&COWProfile::update),
 661     kVoidDest,
 662     SyncOptions::None,
 663     args
 664   );
 665 }
 666
 667 ///////////////////////////////////////////////////////////////////////////////
 668 // Array.
 669
 670 namespace {
 671
 672 VscaledDisp getDictLayoutOffset(IRLS& env, const IRInstruction* inst) {
 673   auto const pos = srcLoc(env, inst, 2).reg();
 674   auto& v = vmain(env);
 675   // We want to index by VanillaDict::Elm but VScaled doesn't let us scale by 24
 676   // So let's use (3 * pos) * 8 to get sizeof(VanillaDict::Elm) * pos
 677   auto pos3 = v.makeReg();
 678   v << lea{pos[pos * 2], pos3};
 679   static_assert(sizeof(VanillaDict::Elm) == 24);
 680   return pos3 * 8 + VanillaDict::dataOff() + VanillaDict::Elm::dataOff();
 681 }
 682
 683 VscaledDisp getKeysetLayoutOffset(IRLS& env, const IRInstruction* inst) {
 684   auto const pos = srcLoc(env, inst, 2).reg();
 685   auto& v = vmain(env);
 686   // We want to index by VanillaDict::Elm but VScaled doesn't let us scale by 16
 687   // So let's use (2 * pos) * 8 to get sizeof(VanillaKeyset::Elm) * pos
 688   auto pos2 = v.makeReg();
 689   v << lea{pos[pos], pos2};
 690   static_assert(sizeof(VanillaKeyset::Elm) == 16);
 691   return pos2 * 8 + VanillaKeyset::dataOff() + VanillaKeyset::tvOff();
 692 }
 693
 694 } // namespace
 695
 696 void cgGetDictPtrIter(IRLS& env, const IRInstruction* inst) {
 697   auto const pos_tmp = inst->src(1);
 698   auto const arr = srcLoc(env, inst, 0).reg();
 699   auto const pos = srcLoc(env, inst, 1).reg();
 700   auto const dst = dstLoc(env, inst, 0).reg();
 701
 702   auto& v = vmain(env);
 703   if (pos_tmp->hasConstVal(TInt)) {
 704     auto const offset = VanillaDict::elmOff(pos_tmp->intVal());
 705     if (deltaFits(offset, sz::dword)) {
 706       v << addqi{safe_cast<int32_t>(offset), arr, dst, v.makeReg()};
 707       return;
 708     }
 709   }
 710
 711   static_assert(sizeof(VanillaDict::Elm) == 24);
 712   auto const px3 = v.makeReg();
 713   v << lea{pos[pos * 2], px3};
 714   v << lea{arr[px3 * 8 + VanillaDict::dataOff()], dst};
 715 }
 716
 717 void cgAdvanceDictPtrIter(IRLS& env, const IRInstruction* inst) {
 718   auto const src = srcLoc(env, inst, 0).reg();
 719   auto const dst = dstLoc(env, inst, 0).reg();
 720
 721   auto& v = vmain(env);
 722   auto const extra = inst->extra<AdvanceDictPtrIter>();
 723   auto const delta = extra->offset * int32_t(sizeof(VanillaDictElm));
 724   v << addqi{delta, src, dst, v.makeReg()};
 725 }
 726
 727 void cgGetKeysetPtrIter(IRLS& env, const IRInstruction* inst) {
 728   auto const pos_tmp = inst->src(1);
 729   auto const arr = srcLoc(env, inst, 0).reg();
 730   auto const pos = srcLoc(env, inst, 1).reg();
 731   auto const dst = dstLoc(env, inst, 0).reg();
 732
 733   auto& v = vmain(env);
 734   if (pos_tmp->hasConstVal(TInt)) {
 735     auto const offset = VanillaKeyset::elmOff(pos_tmp->intVal());
 736     if (deltaFits(offset, sz::dword)) {
 737       v << addqi{safe_cast<int32_t>(offset), arr, dst, v.makeReg()};
 738       return;
 739     }
 740   }
 741
 742   static_assert(sizeof(VanillaKeyset::Elm) == 16);
 743   auto const px16 = v.makeReg();
 744   v << shlqi{4, pos, px16, v.makeReg()};
 745   v << lea{arr[px16 + VanillaKeyset::dataOff()], dst};
 746 }
 747
 748 void cgAdvanceKeysetPtrIter(IRLS& env, const IRInstruction* inst) {
 749   auto const src = srcLoc(env, inst, 0).reg();
 750   auto const dst = dstLoc(env, inst, 0).reg();
 751
 752   auto& v = vmain(env);
 753   auto const extra = inst->extra<AdvanceKeysetPtrIter>();
 754   auto const delta = extra->offset * int32_t(sizeof(VanillaKeysetElm));
 755   v << addqi{delta, src, dst, v.makeReg()};
 756 }
 757
 758 void cgGetVecPtrIter(IRLS& env, const IRInstruction* inst) {
 759   assertx(VanillaVec::stores_unaligned_typed_values);
 760
 761   auto const pos_tmp = inst->src(1);
 762   auto const arr = srcLoc(env, inst, 0).reg();
 763   auto const pos = srcLoc(env, inst, 1).reg();
 764   auto const dst = dstLoc(env, inst, 0).reg();
 765
 766   auto& v = vmain(env);
 767   if (pos_tmp->hasConstVal(TInt)) {
 768     auto const offset = VanillaVec::entryOffset(pos_tmp->intVal());
 769     if (deltaFits(offset.data_offset, sz::dword)) {
 770       v << addqi{safe_cast<int32_t>(offset.data_offset), arr, dst, v.makeReg()};
 771       return;
 772     }
 773   }
 774
 775   static_assert(sizeof(UnalignedTypedValue) == 9);
 776   auto const px9 = v.makeReg();
 777   v << lea{pos[pos * 8], px9};
 778   v << lea{arr[px9 + VanillaVec::entriesOffset()], dst};
 779 }
 780
 781 void cgAdvanceVecPtrIter(IRLS& env, const IRInstruction* inst) {
 782   assertx(VanillaVec::stores_unaligned_typed_values);
 783
 784   auto const src = srcLoc(env, inst, 0).reg();
 785   auto const dst = dstLoc(env, inst, 0).reg();
 786
 787   auto& v = vmain(env);
 788   auto const extra = inst->extra<AdvanceVecPtrIter>();
 789   auto const delta = extra->offset * int32_t(sizeof(UnalignedTypedValue));
 790   v << addqi{delta, src, dst, v.makeReg()};
 791 }
 792
 793 void cgLdPtrIterKey(IRLS& env, const IRInstruction* inst) {
 794   static_assert(sizeof(VanillaDictElm::hash_t) == 4, "");
 795   auto const elm = srcLoc(env, inst, 1).reg();
 796   auto const dst = dstLoc(env, inst, 0);
 797
 798   auto& v = vmain(env);
 799   if (inst->dst(0)->type().needsReg()) {
 800     assertx(dst.hasReg(1));
 801     auto const sf = v.makeReg();
 802     v << cmplim{0, elm[VanillaDictElm::hashOff()], sf};
 803     v << cmovb{CC_L, sf, v.cns(KindOfString), v.cns(KindOfInt64), dst.reg(1)};
 804   }
 805   v << load{elm[VanillaDictElm::keyOff()], dst.reg(0)};
 806 }
 807
 808 void cgLdPtrIterVal(IRLS& env, const IRInstruction* inst) {
 809   static_assert(VanillaDictElm::dataOff() == 0, "");
 810   static_assert(TVOFF(m_data) == 0, "");
 811   auto const elm = srcLoc(env, inst, 1).reg();
 812   loadTV(vmain(env), inst->dst(0), dstLoc(env, inst, 0), elm[0]);
 813 }
 814
 815 void cgEqPtrIter(IRLS& env, const IRInstruction* inst) {
 816   auto const s0 = srcLoc(env, inst, 0).reg();
 817   auto const s1 = srcLoc(env, inst, 1).reg();
 818   auto const d  = dstLoc(env, inst, 0).reg();
 819
 820   auto& v = vmain(env);
 821   auto const sf = v.makeReg();
 822   v << cmpq{s1, s0, sf};
 823   v << setcc{CC_E, sf, d};
 824 }
 825
 826 void cgCheckPtrIterTombstone(IRLS& env, const IRInstruction* inst) {
 827   auto const elm = srcLoc(env, inst, 1).reg();
 828   auto const taken = label(env, inst->taken());
 829
 830   auto& v = vmain(env);
 831   auto constexpr tombstone = static_cast<data_type_t>(kInvalidDataType);
 832   auto const sf = v.makeReg();
 833   v << cmpbim{tombstone, elm[TVOFF(m_type)], sf};
 834   ifThen(v, CC_E, sf, taken);
 835 }
 836
 837 ///////////////////////////////////////////////////////////////////////////////
 838
 839 IMPL_OPCODE_CALL(SetNewElem);
 840 IMPL_OPCODE_CALL(SetNewElemVec);
 841 IMPL_OPCODE_CALL(SetNewElemDict);
 842
 843 IMPL_OPCODE_CALL(AddNewElemVec);
 844 IMPL_OPCODE_CALL(AddNewElemKeyset);
 845
 846 template <TypedValue (*f)(ArrayData*)>
 847 void containerFirstLastHelper(IRLS& env, const IRInstruction* inst) {
 848   auto& v = vmain(env);
 849   cgCallHelper(v, env, CallSpec::direct(f),
 850               callDestTV(env, inst), SyncOptions::None,
 851               argGroup(env, inst).ssa(0));
 852 }
 853
 854 void cgVecFirst(IRLS& env, const IRInstruction* inst) {
 855   containerFirstLastHelper<vecFirstLast<true>>(env, inst);
 856 }
 857
 858 void cgVecLast(IRLS& env, const IRInstruction* inst) {
 859   containerFirstLastHelper<vecFirstLast<false>>(env, inst);
 860 }
 861
 862 void cgDictFirst(IRLS& env, const IRInstruction* inst) {
 863   containerFirstLastHelper<arrFirstLast<true, false>>(env, inst);
 864 }
 865
 866 void cgDictLast(IRLS& env, const IRInstruction* inst) {
 867   containerFirstLastHelper<arrFirstLast<false, false>>(env, inst);
 868 }
 869
 870 void cgDictFirstKey(IRLS& env, const IRInstruction* inst) {
 871   containerFirstLastHelper<arrFirstLast<true, true>>(env, inst);
 872 }
 873
 874 void cgDictLastKey(IRLS& env, const IRInstruction* inst) {
 875   containerFirstLastHelper<arrFirstLast<false, true>>(env, inst);
 876 }
 877
 878 void cgKeysetFirst(IRLS& env, const IRInstruction* inst) {
 879   containerFirstLastHelper<arrFirstLast<true, false>>(env, inst);
 880 }
 881
 882 void cgKeysetLast(IRLS& env, const IRInstruction* inst) {
 883   containerFirstLastHelper<arrFirstLast<false, false>>(env, inst);
 884 }
 885
 886 ///////////////////////////////////////////////////////////////////////////////
 887 // Vec.
 888
 889 irlower::LvalPtrs implVecElemLval(IRLS& env, Vreg rarr,
 890                                   Vreg ridx, const SSATmp* idx) {
 891   auto& v = vmain(env);
 892
 893   static_assert(sizeof(TypedValue) == 16, "");
 894   static_assert(TVOFF(m_data) == 0, "");
 895
 896   if (idx && idx->hasConstVal()) {
 897     auto const offset = VanillaVec::entryOffset(idx->intVal());
 898     if (deltaFits(offset.type_offset, sz::dword) &&
 899         deltaFits(offset.data_offset, sz::dword)) {
 900       return {rarr[offset.type_offset], rarr[offset.data_offset]};
 901     }
 902   }
 903
 904   if constexpr (VanillaVec::stores_unaligned_typed_values) {
 905     // Compute `rarr + ridx * sizeof(UnalignedTypedValue) + VanillaVec::entriesOffset().
 906     static_assert(IMPLIES(VanillaVec::stores_unaligned_typed_values, sizeof(UnalignedTypedValue) == 9));
 907     auto ridx_times_9 = v.makeReg();
 908     v << lea{ridx[ridx * 8], ridx_times_9};
 909     auto const type_offset = VanillaVec::entriesOffset() + offsetof(UnalignedTypedValue, m_type);
 910     auto const data_offset = VanillaVec::entriesOffset() + offsetof(UnalignedTypedValue, m_data);
 911     return {rarr[ridx_times_9] + type_offset, rarr[ridx_times_9] + data_offset};
 912   } else {
 913     // See PackedBlock::LvalAt for an explanation of this math.
 914     auto const x = v.makeReg();
 915     auto const a = v.makeReg();
 916     auto const b = v.makeReg();
 917     v << andqi{-8, ridx, x, v.makeReg()};
 918     v << lea{rarr[x    * 8] + VanillaVec::entriesOffset(), a};
 919     v << lea{rarr[ridx * 8] + VanillaVec::entriesOffset(), b};
 920     return {a[ridx], b[x] + 8};
 921   }
 922 }
 923
 924 /*
 925  * Thread-local RDS packed array access sampling counter.
 926  */
 927 rds::Link<uint32_t, rds::Mode::Local> s_counter;
 928
 929 void cgLdVecElemAddr(IRLS& env, const IRInstruction* inst) {
 930   auto const arr = srcLoc(env, inst, 0).reg();
 931   auto const idx = srcLoc(env, inst, 1).reg();
 932   auto const dstLoc = irlower::dstLoc(env, inst, 0);
 933   auto const addr = implVecElemLval(env, arr, idx, inst->src(1));
 934   vmain(env) << lea{addr.val, dstLoc.reg(tv_lval::val_idx)};
 935   vmain(env) << lea{addr.type, dstLoc.reg(tv_lval::type_idx)};
 936 }
 937
 938 void cgLdVecElem(IRLS& env, const IRInstruction* inst) {
 939   auto const arr = srcLoc(env, inst, 0).reg();
 940   auto const idx = srcLoc(env, inst, 1).reg();
 941   auto const addr = implVecElemLval(env, arr, idx, inst->src(1));
 942
 943   loadTV(vmain(env), inst->dst()->type(), dstLoc(env, inst, 0),
 944          addr.type, addr.val);
 945 }
 946
 947 void cgReserveVecNewElem(IRLS& env, const IRInstruction* i) {
 948   static_assert(ArrayData::sizeofSize() == 4, "");
 949
 950   auto& v = vmain(env);
 951   auto arrayData = srcLoc(env, i, 0).reg();
 952   auto const sizePtr = arrayData[ArrayData::offsetofSize()];
 953
 954   // If the check below succeeds, we'll end up returning the original size
 955   // so just use the destination register to hold the orignal size
 956   auto const size = dstLoc(env, i, 0).reg();
 957   v << loadzlq{sizePtr, size};
 958
 959   { // Bail out unless size < cap
 960     auto const indexb = v.makeReg();
 961     v << loadb{arrayData[VanillaVec::SizeIndexOffset], indexb};
 962     auto const index = v.makeReg();
 963     v << movzbq{indexb, index};
 964     auto const cap = v.makeReg();
 965     auto const table =
 966       reinterpret_cast<uintptr_t>(kSizeIndex2VanillaVecCapacity);
 967     if (table < std::numeric_limits<int>::max()) {
 968       v << loadzlq{baseless(index * 4 + table), cap};
 969     } else {
 970       auto const base = v.cns(table);
 971       v << loadzlq{base[index * 4], cap};
 972     }
 973
 974     auto const sf = v.makeReg();
 975     v << cmpq{size, cap, sf};
 976     ifThen(v, CC_BE, sf, label(env, i->taken()));
 977   }
 978
 979   v << inclm{sizePtr, v.makeReg()};
 980 }
 981
 982 ///////////////////////////////////////////////////////////////////////////////
 983 // Dict.
 984
 985 namespace {
 986
 987 void implDictGet(IRLS& env, const IRInstruction* inst, MOpMode mode) {
 988   assertx(mode == MOpMode::None || mode == MOpMode::Warn);
 989   BUILD_OPTAB(DICTGET_HELPER_TABLE, getKeyType(inst->src(1)), mode);
 990   auto const sync = mode == MOpMode::None
 991     ? SyncOptions::None
 992     : SyncOptions::Sync;
 993   auto const args = argGroup(env, inst).ssa(0).ssa(1);
 994
 995   auto& v = vmain(env);
 996   cgCallHelper(v, env, target, callDestTV(env, inst), sync, args);
 997 }
 998
 999 void implDictSet(IRLS& env, const IRInstruction* inst) {
1000   BUILD_OPTAB(DICTSET_HELPER_TABLE, getKeyType(inst->src(1)));
1001   auto const args = argGroup(env, inst).ssa(0).ssa(1).typedValue(2);
1002
1003   auto& v = vmain(env);
1004   cgCallHelper(v, env, target, callDest(env, inst), SyncOptions::Sync, args);
1005 }
1006
1007 }
1008
1009 void cgElemDictD(IRLS& env, const IRInstruction* inst) {
1010   BUILD_OPTAB(ELEM_DICT_D_HELPER_TABLE, getKeyType(inst->src(1)));
1011   auto const args = argGroup(env, inst).ssa(0).ssa(1);
1012
1013   auto& v = vmain(env);
1014   cgCallHelper(v, env, target, callDest(env, inst), SyncOptions::Sync, args);
1015 }
1016
1017 void cgElemDictU(IRLS& env, const IRInstruction* inst) {
1018   BUILD_OPTAB(ELEM_DICT_U_HELPER_TABLE, getKeyType(inst->src(1)));
1019   auto const args = argGroup(env, inst).ssa(0).ssa(1);
1020
1021   auto& v = vmain(env);
1022   cgCallHelper(v, env, target, callDest(env, inst), SyncOptions::Sync, args);
1023 }
1024
1025 void cgElemDictK(IRLS& env, const IRInstruction* inst) {
1026   auto const dict = srcLoc(env, inst, 0).reg();
1027   auto const dst = dstLoc(env, inst, 0);
1028
1029   auto& v = vmain(env);
1030   auto const off = getDictLayoutOffset(env, inst);
1031
1032   v << lea{dict[off], dst.reg(tv_lval::val_idx)};
1033   static_assert(TVOFF(m_data) == 0, "");
1034   v << lea{dict[off + TVOFF(m_type)], dst.reg(tv_lval::type_idx)};
1035 }
1036
1037 void cgDictGet(IRLS& env, const IRInstruction* inst) {
1038   implDictGet(env, inst, MOpMode::Warn);
1039 }
1040 void cgDictGetQuiet(IRLS& env, const IRInstruction* inst) {
1041   implDictGet(env, inst, MOpMode::None);
1042 }
1043
1044 void cgDictGetK(IRLS& env, const IRInstruction* inst) {
1045   auto const dict = srcLoc(env, inst, 0).reg();
1046   auto const off = getDictLayoutOffset(env, inst);
1047   loadTV(vmain(env), inst->dst(0), dstLoc(env, inst, 0), dict[off]);
1048 }
1049
1050 void cgDictSet(IRLS& env, const IRInstruction* i)    { implDictSet(env, i); }
1051
1052 void cgDictIsset(IRLS& env, const IRInstruction* inst) {
1053   auto const key     = inst->src(1);
1054   BUILD_OPTAB(DICT_ISSET_ELEM_HELPER_TABLE, getKeyType(key));
1055
1056   auto args = argGroup(env, inst).ssa(0).ssa(1);
1057
1058   auto& v = vmain(env);
1059   cgCallHelper(v, env, target, callDest(env, inst), SyncOptions::None, args);
1060 }
1061
1062 void cgDictIdx(IRLS& env, const IRInstruction* inst) {
1063   auto const def = inst->src(2);
1064   if (def->isA(TInitNull)) return implDictGet(env, inst, MOpMode::None);
1065   auto const keyType = getKeyType(inst->src(1));
1066
1067   auto const target = [&] () -> CallSpec {
1068     if (keyType == KeyType::Int) return CallSpec::direct(dictIdxI);
1069     assertx(keyType == KeyType::Str);
1070     auto const scan =
1071       inst->extra<SizeHintData>()->hint == SizeHintData::SmallStatic &&
1072       inst->src(1)->isA(TStaticStr);
1073     return CallSpec::direct(scan ? dictIdxScan : dictIdxS);
1074   }();
1075
1076   auto args = argGroup(env, inst).ssa(0).ssa(1).typedValue(2);
1077   auto& v = vmain(env);
1078   cgCallHelper(v, env, target, callDestTV(env, inst),
1079                SyncOptions::None, args);
1080 }
1081
1082 ///////////////////////////////////////////////////////////////////////////////
1083 // Keyset.
1084
1085 namespace {
1086
1087 void implKeysetGet(IRLS& env, const IRInstruction* inst) {
1088   auto const key = inst->src(1);
1089   auto const mode = inst->op() == KeysetGetQuiet
1090     ? MOpMode::None
1091     : MOpMode::Warn;
1092   auto const sync = inst->op() == KeysetGetQuiet
1093     ? SyncOptions::None
1094     : SyncOptions::Sync;
1095   BUILD_OPTAB(KEYSETGET_HELPER_TABLE, getKeyType(key), mode);
1096
1097   auto args = argGroup(env, inst).ssa(0).ssa(1);
1098
1099   auto& v = vmain(env);
1100   cgCallHelper(v, env, target, callDestTV(env, inst), sync, args);
1101 }
1102
1103 }
1104
1105 void cgKeysetGet(IRLS& env, const IRInstruction* inst) {
1106   implKeysetGet(env, inst);
1107 }
1108 void cgKeysetGetQuiet(IRLS& env, const IRInstruction* inst) {
1109   implKeysetGet(env, inst);
1110 }
1111
1112 void cgKeysetGetK(IRLS& env, const IRInstruction* inst) {
1113   auto const keyset = srcLoc(env, inst, 0).reg();
1114   auto const off = getKeysetLayoutOffset(env, inst);
1115   loadTV(vmain(env), inst->dst(0), dstLoc(env, inst, 0), keyset[off]);
1116 }
1117
1118 void cgSetNewElemKeyset(IRLS& env, const IRInstruction* inst) {
1119   auto const key     = inst->src(1);
1120   BUILD_OPTAB(KEYSET_SETNEWELEM_HELPER_TABLE, getKeyType(key));
1121
1122   auto args = argGroup(env, inst).ssa(0).ssa(1);
1123
1124   auto& v = vmain(env);
1125   cgCallHelper(v, env, target, callDest(env, inst), SyncOptions::Sync, args);
1126 }
1127
1128 void cgKeysetIsset(IRLS& env, const IRInstruction* inst) {
1129   auto const key = inst->src(1);
1130   BUILD_OPTAB(KEYSET_ISSET_ELEM_HELPER_TABLE, getKeyType(key));
1131
1132   auto args = argGroup(env, inst).ssa(0).ssa(1);
1133
1134   auto& v = vmain(env);
1135   cgCallHelper(v, env, target, callDest(env, inst), SyncOptions::None, args);
1136 }
1137
1138 void cgKeysetIdx(IRLS& env, const IRInstruction* inst) {
1139   auto const key     = inst->src(1);
1140   auto const target  = getKeyType(key) == KeyType::Int
1141     ? CallSpec::direct(keysetIdxI)
1142     : CallSpec::direct(keysetIdxS);
1143   auto args = argGroup(env, inst).ssa(0).ssa(1).typedValue(2);
1144   auto& v = vmain(env);
1145   cgCallHelper(v, env, target, callDestTV(env, inst),
1146                SyncOptions::None, args);
1147 }
1148
1149 ///////////////////////////////////////////////////////////////////////////////
1150 // Collections.
1151
1152 IMPL_OPCODE_CALL(PairIsset);
1153 IMPL_OPCODE_CALL(VectorIsset);
1154
1155 void cgVectorSet(IRLS& env, const IRInstruction* inst) {
1156   auto const target = inst->src(1)->isA(TInt)
1157     ? CallSpec::direct(MInstrHelpers::vectorSetImplI)
1158     : CallSpec::direct(MInstrHelpers::vectorSetImplS);
1159
1160   auto const args = argGroup(env, inst)
1161     .ssa(0)
1162     .ssa(1)
1163     .typedValue(2);
1164
1165   auto& v = vmain(env);
1166   cgCallHelper(v, env, target, kVoidDest, SyncOptions::Sync, args);
1167 }
1168
1169 void cgMapGet(IRLS& env, const IRInstruction* inst) {
1170   auto const target = inst->src(1)->isA(TInt)
1171     ? CallSpec::direct(MInstrHelpers::mapGetImpl<KeyType::Int>)
1172     : CallSpec::direct(MInstrHelpers::mapGetImpl<KeyType::Str>);
1173
1174   auto const args = argGroup(env, inst).ssa(0).ssa(1);
1175
1176   auto& v = vmain(env);
1177   cgCallHelper(v, env, target, callDestTV(env, inst),
1178                SyncOptions::Sync, args);
1179 }
1180
1181 void cgMapSet(IRLS& env, const IRInstruction* inst) {
1182   auto const target = inst->src(1)->isA(TInt)
1183     ? CallSpec::direct(MInstrHelpers::mapSetImpl<KeyType::Int>)
1184     : CallSpec::direct(MInstrHelpers::mapSetImpl<KeyType::Str>);
1185
1186   auto const args = argGroup(env, inst)
1187     .ssa(0)
1188     .ssa(1)
1189     .typedValue(2);
1190
1191   auto& v = vmain(env);
1192   cgCallHelper(v, env, target, kVoidDest, SyncOptions::Sync, args);
1193 }
1194
1195 void cgMapIsset(IRLS& env, const IRInstruction* inst) {
1196   auto const target = inst->src(1)->isA(TInt)
1197     ? CallSpec::direct(MInstrHelpers::mapIssetImpl<KeyType::Int>)
1198     : CallSpec::direct(MInstrHelpers::mapIssetImpl<KeyType::Str>);
1199
1200   auto const args = argGroup(env, inst).ssa(0).ssa(1);
1201
1202   auto& v = vmain(env);
1203   cgCallHelper(v, env, target, callDest(env, inst),
1204                SyncOptions::None, args);
1205 }
1206
1207 ///////////////////////////////////////////////////////////////////////////////
1208
1209 }