2 +----------------------------------------------------------------------+
4 +----------------------------------------------------------------------+
5 | Copyright (c) 2010-2014 Facebook, Inc. (http://www.facebook.com) |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
17 #include "hphp/runtime/vm/jit/code-gen-arm.h"
20 #include "folly/Optional.h"
22 #include "hphp/runtime/ext/ext_continuation.h"
23 #include "hphp/runtime/vm/jit/abi-arm.h"
24 #include "hphp/runtime/vm/jit/arg-group.h"
25 #include "hphp/runtime/vm/jit/code-gen-helpers-arm.h"
26 #include "hphp/runtime/vm/jit/jump-smash.h"
27 #include "hphp/runtime/vm/jit/native-calls.h"
28 #include "hphp/runtime/vm/jit/reg-algorithms.h"
29 #include "hphp/runtime/vm/jit/service-requests-arm.h"
30 #include "hphp/runtime/vm/jit/translator-inline.h"
32 namespace HPHP
{ namespace JIT
{ namespace ARM
{
38 //////////////////////////////////////////////////////////////////////
40 #define NOOP_OPCODE(name) void CodeGenerator::cg##name(IRInstruction*) {}
45 NOOP_OPCODE(AssertLoc
)
46 NOOP_OPCODE(AssertStk
)
49 NOOP_OPCODE(ExceptionBarrier
)
50 NOOP_OPCODE(TakeStack
)
53 NOOP_OPCODE(DbgAssertPtr
);
55 // When implemented this shouldn't be a nop, but there's no reason to make us
56 // punt on everything until then.
57 NOOP_OPCODE(DbgAssertRetAddr
)
61 //////////////////////////////////////////////////////////////////////
63 #define CALL_OPCODE(name) \
64 void CodeGenerator::cg##name(IRInstruction* i) { cgCallNative(m_as, i); }
67 CALL_OPCODE(ConvIntToStr
)
70 CALL_OPCODE(NewPackedArray
)
72 CALL_OPCODE(ConcatStrStr
)
73 CALL_OPCODE(ConcatIntStr
)
74 CALL_OPCODE(ConcatStrInt
)
78 CALL_OPCODE(PrintBool
)
80 CALL_OPCODE(AddElemStrKey
)
82 CALL_OPCODE(ConvBoolToArr
)
83 CALL_OPCODE(ConvDblToArr
)
84 CALL_OPCODE(ConvIntToArr
)
85 CALL_OPCODE(ConvObjToArr
)
86 CALL_OPCODE(ConvStrToArr
)
87 CALL_OPCODE(ConvCellToArr
)
89 CALL_OPCODE(ConvStrToBool
)
90 CALL_OPCODE(ConvCellToBool
)
91 CALL_OPCODE(ConvArrToDbl
)
92 CALL_OPCODE(ConvObjToDbl
)
93 CALL_OPCODE(ConvStrToDbl
)
94 CALL_OPCODE(ConvCellToDbl
)
96 CALL_OPCODE(ConvObjToInt
)
97 CALL_OPCODE(ConvArrToInt
)
98 CALL_OPCODE(ConvStrToInt
)
100 CALL_OPCODE(RaiseWarning
)
101 CALL_OPCODE(RaiseError
)
102 CALL_OPCODE(ConvCellToObj
)
103 CALL_OPCODE(LookupClsMethod
)
104 CALL_OPCODE(RaiseNotice
)
105 CALL_OPCODE(LookupClsRDSHandle
)
106 CALL_OPCODE(LdSwitchStrIndex
)
107 CALL_OPCODE(LdSwitchDblIndex
)
108 CALL_OPCODE(LdSwitchObjIndex
)
109 CALL_OPCODE(CustomInstanceInit
)
110 CALL_OPCODE(LdClsCtor
)
112 CALL_OPCODE(LdArrFuncCtx
)
113 CALL_OPCODE(LdArrFPushCuf
)
114 CALL_OPCODE(LdStrFPushCuf
)
115 CALL_OPCODE(NewArray
)
118 CALL_OPCODE(ClosureStaticLocInit
)
119 CALL_OPCODE(VerifyParamCallable
)
120 CALL_OPCODE(VerifyParamFail
)
121 CALL_OPCODE(WarnNonObjProp
)
122 CALL_OPCODE(ThrowNonObjProp
)
123 CALL_OPCODE(RaiseUndefProp
)
124 CALL_OPCODE(AddNewElem
)
125 CALL_OPCODE(ColAddElemC
)
126 CALL_OPCODE(ColAddNewElemC
)
127 CALL_OPCODE(ArrayAdd
)
128 CALL_OPCODE(CreateContFunc
)
129 CALL_OPCODE(CreateContMeth
)
130 CALL_OPCODE(CreateAFWHFunc
)
131 CALL_OPCODE(CreateAFWHMeth
)
132 CALL_OPCODE(CreateSRWH
)
133 CALL_OPCODE(TypeProfileFunc
)
134 CALL_OPCODE(IncStatGrouped
)
136 /////////////////////////////////////////////////////////////////////
137 void cgPunt(const char* file
, int line
, const char* func
, uint32_t bcOff
,
138 const Func
* vmFunc
) {
139 FTRACE(1, "punting: {}\n", func
);
140 throw FailedCodeGen(file
, line
, func
, bcOff
, vmFunc
);
143 #define PUNT_OPCODE(name) \
144 void CodeGenerator::cg##name(IRInstruction* inst) { \
145 cgPunt(__FILE__, __LINE__, #name, m_curInst->marker().bcOff, \
149 #define CG_PUNT(instr) \
150 cgPunt(__FILE__, __LINE__, #instr, m_curInst->marker().bcOff, curFunc())
152 /////////////////////////////////////////////////////////////////////
153 //TODO t3702757: Convert to CALL_OPCODE, the following set works on
154 // x86 but needs a closer look on arm
155 PUNT_OPCODE(AddElemIntKey
)
156 PUNT_OPCODE(ConvCellToInt
)
157 PUNT_OPCODE(ArrayIdx
)
158 PUNT_OPCODE(RaiseArrayIndexNotice
)
159 PUNT_OPCODE(RaiseUninitLoc
)
160 PUNT_OPCODE(VerifyRetCallable
)
161 PUNT_OPCODE(VerifyRetFail
)
162 PUNT_OPCODE(GenericIdx
)
163 // End of failing set
164 /////////////////////////////////////////////////////////////////////
166 PUNT_OPCODE(ConvArrToBool
)
167 PUNT_OPCODE(ConvDblToBool
)
168 PUNT_OPCODE(ConvIntToBool
)
169 PUNT_OPCODE(ConvObjToBool
)
170 PUNT_OPCODE(ConvBoolToDbl
)
171 PUNT_OPCODE(ConvIntToDbl
)
173 PUNT_OPCODE(ConvBoolToInt
)
174 PUNT_OPCODE(ConvDblToInt
)
176 PUNT_OPCODE(ConvBoolToStr
)
177 PUNT_OPCODE(ConvDblToStr
)
178 PUNT_OPCODE(ConvObjToStr
)
179 PUNT_OPCODE(ConvResToStr
)
180 PUNT_OPCODE(ConvCellToStr
)
182 PUNT_OPCODE(CheckTypeMem
)
183 PUNT_OPCODE(CheckLoc
)
185 PUNT_OPCODE(CoerceStk
)
186 PUNT_OPCODE(CheckDefinedClsEq
)
187 PUNT_OPCODE(TryEndCatch
)
188 PUNT_OPCODE(LdUnwinderValue
)
189 PUNT_OPCODE(DeleteUnwinderException
)
198 PUNT_OPCODE(ExtendsClass
)
199 PUNT_OPCODE(IsWaitHandle
)
200 PUNT_OPCODE(InstanceOf
)
201 PUNT_OPCODE(InstanceOfIface
)
202 PUNT_OPCODE(InterfaceSupportsArr
)
203 PUNT_OPCODE(InterfaceSupportsStr
)
204 PUNT_OPCODE(InterfaceSupportsInt
)
205 PUNT_OPCODE(InterfaceSupportsDbl
)
206 PUNT_OPCODE(IsTypeMem
)
207 PUNT_OPCODE(IsNTypeMem
)
224 PUNT_OPCODE(InstanceOfBitmask
)
225 PUNT_OPCODE(NInstanceOfBitmask
)
227 PUNT_OPCODE(IsScalarType
)
235 PUNT_OPCODE(JmpGtInt
)
236 PUNT_OPCODE(JmpGteInt
)
237 PUNT_OPCODE(JmpLtInt
)
238 PUNT_OPCODE(JmpLteInt
)
239 PUNT_OPCODE(JmpEqInt
)
240 PUNT_OPCODE(JmpNeqInt
)
242 PUNT_OPCODE(JmpNSame
)
243 PUNT_OPCODE(JmpInstanceOfBitmask
)
244 PUNT_OPCODE(JmpNInstanceOfBitmask
)
246 PUNT_OPCODE(JmpNZero
)
247 PUNT_OPCODE(ReqBindJmpGt
)
248 PUNT_OPCODE(ReqBindJmpGte
)
249 PUNT_OPCODE(ReqBindJmpLt
)
250 PUNT_OPCODE(ReqBindJmpLte
)
251 PUNT_OPCODE(ReqBindJmpEq
)
252 PUNT_OPCODE(ReqBindJmpNeq
)
253 PUNT_OPCODE(ReqBindJmpGtInt
)
254 PUNT_OPCODE(ReqBindJmpGteInt
)
255 PUNT_OPCODE(ReqBindJmpLtInt
)
256 PUNT_OPCODE(ReqBindJmpLteInt
)
257 PUNT_OPCODE(ReqBindJmpEqInt
)
258 PUNT_OPCODE(ReqBindJmpNeqInt
)
259 PUNT_OPCODE(ReqBindJmpSame
)
260 PUNT_OPCODE(ReqBindJmpNSame
)
261 PUNT_OPCODE(ReqBindJmpInstanceOfBitmask
)
262 PUNT_OPCODE(ReqBindJmpNInstanceOfBitmask
)
263 PUNT_OPCODE(ReqBindJmpZero
)
264 PUNT_OPCODE(ReqBindJmpNZero
)
265 PUNT_OPCODE(SideExitJmpGt
)
266 PUNT_OPCODE(SideExitJmpGte
)
267 PUNT_OPCODE(SideExitJmpLt
)
268 PUNT_OPCODE(SideExitJmpLte
)
269 PUNT_OPCODE(SideExitJmpEq
)
270 PUNT_OPCODE(SideExitJmpNeq
)
271 PUNT_OPCODE(SideExitJmpGtInt
)
272 PUNT_OPCODE(SideExitJmpGteInt
)
273 PUNT_OPCODE(SideExitJmpLtInt
)
274 PUNT_OPCODE(SideExitJmpLteInt
)
275 PUNT_OPCODE(SideExitJmpEqInt
)
276 PUNT_OPCODE(SideExitJmpNeqInt
)
277 PUNT_OPCODE(SideExitJmpSame
)
278 PUNT_OPCODE(SideExitJmpNSame
)
279 PUNT_OPCODE(SideExitJmpInstanceOfBitmask
)
280 PUNT_OPCODE(SideExitJmpNInstanceOfBitmask
)
281 PUNT_OPCODE(SideExitJmpZero
)
282 PUNT_OPCODE(SideExitJmpNZero
)
283 PUNT_OPCODE(SideExitGuardLoc
)
284 PUNT_OPCODE(JmpIndirect
)
285 PUNT_OPCODE(CheckSurpriseFlags
)
286 PUNT_OPCODE(SurpriseHook
)
287 PUNT_OPCODE(FunctionExitSurpriseHook
)
288 PUNT_OPCODE(ExitOnVarEnv
)
289 PUNT_OPCODE(ReleaseVVOrExit
)
290 PUNT_OPCODE(CheckInit
)
291 PUNT_OPCODE(CheckInitMem
)
292 PUNT_OPCODE(CheckCold
)
293 PUNT_OPCODE(CheckNullptr
)
294 PUNT_OPCODE(CheckBounds
)
295 PUNT_OPCODE(LdVectorSize
)
296 PUNT_OPCODE(CheckPackedArrayBounds
)
297 PUNT_OPCODE(CheckPackedArrayElemNull
)
298 PUNT_OPCODE(VectorHasFrozenCopy
)
299 PUNT_OPCODE(VectorDoCow
)
300 PUNT_OPCODE(CheckNonNull
)
301 PUNT_OPCODE(AssertNonNull
)
303 PUNT_OPCODE(UnboxPtr
)
305 PUNT_OPCODE(LdVectorBase
)
306 PUNT_OPCODE(LdPairBase
)
307 PUNT_OPCODE(LdLocAddr
)
311 PUNT_OPCODE(LdPackedArrayElem
)
314 PUNT_OPCODE(LdRetAddr
)
315 PUNT_OPCODE(ConvClsToCctx
)
319 PUNT_OPCODE(LdClsCached
)
320 PUNT_OPCODE(LdClsCachedSafe
)
321 PUNT_OPCODE(LdClsCtx
)
322 PUNT_OPCODE(LdClsCctx
)
323 PUNT_OPCODE(LdClsCns
)
324 PUNT_OPCODE(LookupClsCns
)
326 PUNT_OPCODE(LdClsInitData
)
327 PUNT_OPCODE(LdClsStaticInitData
)
328 PUNT_OPCODE(LookupCns
)
329 PUNT_OPCODE(LookupCnsE
)
330 PUNT_OPCODE(LookupCnsU
)
331 PUNT_OPCODE(DerefClsRDSHandle
)
332 PUNT_OPCODE(LookupClsMethodCache
)
333 PUNT_OPCODE(LdClsMethodCacheFunc
)
334 PUNT_OPCODE(LdClsMethodCacheCls
)
335 PUNT_OPCODE(LdClsMethodFCacheFunc
)
336 PUNT_OPCODE(LookupClsMethodFCache
)
337 PUNT_OPCODE(GetCtxFwdCallDyn
);
338 PUNT_OPCODE(GetCtxFwdCall
)
339 PUNT_OPCODE(LdClsMethod
)
340 PUNT_OPCODE(LdPropAddr
)
341 PUNT_OPCODE(LdClsPropAddr
)
342 PUNT_OPCODE(LdClsPropAddrCached
)
343 PUNT_OPCODE(LdObjMethod
)
344 PUNT_OPCODE(LdObjInvoke
)
345 PUNT_OPCODE(LdGblAddrDef
)
346 PUNT_OPCODE(LdGblAddr
)
347 PUNT_OPCODE(LdObjClass
)
349 PUNT_OPCODE(LdFuncCachedU
)
350 PUNT_OPCODE(LdFuncCachedSafe
)
351 PUNT_OPCODE(LdSSwitchDestFast
)
352 PUNT_OPCODE(LdSSwitchDestSlow
)
353 PUNT_OPCODE(JmpSwitchDest
)
354 PUNT_OPCODE(ConstructInstance
)
355 PUNT_OPCODE(InitProps
)
356 PUNT_OPCODE(InitSProps
)
357 PUNT_OPCODE(NewInstanceRaw
)
358 PUNT_OPCODE(InitObjProps
)
359 PUNT_OPCODE(StClosureFunc
)
360 PUNT_OPCODE(StClosureArg
)
361 PUNT_OPCODE(StClosureCtx
)
362 PUNT_OPCODE(NewStructArray
)
363 PUNT_OPCODE(FreeActRec
)
364 PUNT_OPCODE(CallArray
)
365 PUNT_OPCODE(NativeImpl
)
367 PUNT_OPCODE(StRetVal
)
368 PUNT_OPCODE(RetAdjustStack
)
375 PUNT_OPCODE(IterCopy
)
376 PUNT_OPCODE(LdStaticLocCached
)
377 PUNT_OPCODE(CheckStaticLocInit
)
378 PUNT_OPCODE(StaticLocInitCached
)
379 PUNT_OPCODE(CufIterSpillFrame
)
380 PUNT_OPCODE(ReqRetranslateOpt
)
383 PUNT_OPCODE(IncRefCtx
)
384 PUNT_OPCODE(DecRefThis
)
385 PUNT_OPCODE(GenericRetDecRefs
)
387 PUNT_OPCODE(DecRefNZ
)
388 PUNT_OPCODE(DefInlineFP
)
389 PUNT_OPCODE(InlineReturn
)
390 PUNT_OPCODE(DefInlineSP
)
392 PUNT_OPCODE(ThingExists
);
395 PUNT_OPCODE(StashGeneratorSP
)
396 PUNT_OPCODE(ReDefGeneratorSP
)
397 PUNT_OPCODE(VerifyParamCls
)
398 PUNT_OPCODE(VerifyRetCls
)
399 PUNT_OPCODE(ConcatCellCell
)
400 PUNT_OPCODE(AKExists
)
401 PUNT_OPCODE(ContEnter
)
402 PUNT_OPCODE(ContPreNext
)
403 PUNT_OPCODE(ContStartedCheck
)
404 PUNT_OPCODE(ContSetRunning
)
405 PUNT_OPCODE(ContValid
)
406 PUNT_OPCODE(ContArIncKey
)
407 PUNT_OPCODE(ContArUpdateIdx
)
408 PUNT_OPCODE(LdContActRec
)
409 PUNT_OPCODE(StContArRaw
)
410 PUNT_OPCODE(LdContArValue
)
411 PUNT_OPCODE(StContArValue
)
412 PUNT_OPCODE(LdContArKey
)
413 PUNT_OPCODE(StContArKey
)
414 PUNT_OPCODE(LdWHState
)
415 PUNT_OPCODE(LdWHResult
)
416 PUNT_OPCODE(LdAFWHActRec
)
417 PUNT_OPCODE(IterInit
)
418 PUNT_OPCODE(IterInitK
)
419 PUNT_OPCODE(IterNext
)
420 PUNT_OPCODE(IterNextK
)
421 PUNT_OPCODE(WIterInit
)
422 PUNT_OPCODE(WIterInitK
)
423 PUNT_OPCODE(WIterNext
)
424 PUNT_OPCODE(WIterNextK
)
425 PUNT_OPCODE(MIterInit
)
426 PUNT_OPCODE(MIterInitK
)
427 PUNT_OPCODE(MIterNext
)
428 PUNT_OPCODE(MIterNextK
)
429 PUNT_OPCODE(IterFree
)
430 PUNT_OPCODE(MIterFree
)
431 PUNT_OPCODE(DecodeCufIter
)
432 PUNT_OPCODE(CIterFree
)
433 PUNT_OPCODE(DefMIStateBase
)
437 PUNT_OPCODE(PropDXStk
)
438 PUNT_OPCODE(CGetProp
)
439 PUNT_OPCODE(VGetProp
)
440 PUNT_OPCODE(VGetPropStk
)
441 PUNT_OPCODE(BindProp
)
442 PUNT_OPCODE(BindPropStk
)
444 PUNT_OPCODE(SetPropStk
)
445 PUNT_OPCODE(UnsetProp
)
446 PUNT_OPCODE(SetOpProp
)
447 PUNT_OPCODE(SetOpPropStk
)
448 PUNT_OPCODE(IncDecProp
)
449 PUNT_OPCODE(IncDecPropStk
)
450 PUNT_OPCODE(EmptyProp
)
451 PUNT_OPCODE(IssetProp
)
453 PUNT_OPCODE(ElemArray
)
455 PUNT_OPCODE(ElemDXStk
)
457 PUNT_OPCODE(ElemUXStk
)
458 PUNT_OPCODE(ArrayGet
)
459 PUNT_OPCODE(StringGet
)
461 PUNT_OPCODE(CGetElem
)
462 PUNT_OPCODE(VGetElem
)
463 PUNT_OPCODE(VGetElemStk
)
464 PUNT_OPCODE(BindElem
)
465 PUNT_OPCODE(BindElemStk
)
466 PUNT_OPCODE(ArraySet
)
468 PUNT_OPCODE(ArraySetRef
)
470 PUNT_OPCODE(SetElemStk
)
471 PUNT_OPCODE(SetWithRefElem
)
472 PUNT_OPCODE(SetWithRefElemStk
)
473 PUNT_OPCODE(UnsetElem
)
474 PUNT_OPCODE(UnsetElemStk
)
475 PUNT_OPCODE(SetOpElem
)
476 PUNT_OPCODE(SetOpElemStk
)
477 PUNT_OPCODE(IncDecElem
)
478 PUNT_OPCODE(IncDecElemStk
)
479 PUNT_OPCODE(SetNewElem
)
480 PUNT_OPCODE(SetNewElemStk
)
481 PUNT_OPCODE(SetNewElemArray
)
482 PUNT_OPCODE(SetNewElemArrayStk
)
483 PUNT_OPCODE(SetWithRefNewElem
)
484 PUNT_OPCODE(SetWithRefNewElemStk
)
485 PUNT_OPCODE(BindNewElem
)
486 PUNT_OPCODE(BindNewElemStk
)
487 PUNT_OPCODE(ArrayIsset
)
488 PUNT_OPCODE(StringIsset
)
489 PUNT_OPCODE(VectorIsset
)
490 PUNT_OPCODE(PairIsset
)
491 PUNT_OPCODE(MapIsset
)
492 PUNT_OPCODE(IssetElem
)
493 PUNT_OPCODE(EmptyElem
)
496 PUNT_OPCODE(IncTransCounter
)
497 PUNT_OPCODE(IncProfCounter
)
498 PUNT_OPCODE(DbgAssertType
)
505 //////////////////////////////////////////////////////////////////////
507 static TCA kEndOfTargetChain
= reinterpret_cast<TCA
>(0xf00ffeeffaaff11f);
509 void CodeGenerator::emitJumpToBlock(CodeBlock
& cb
,
512 vixl::MacroAssembler as
{ cb
};
514 if (m_state
.addresses
[target
]) {
518 // The block hasn't been emitted yet. Record the location in CodegenState.
519 // CodegenState holds a map from Block* to the head of a linked list, where
520 // the jump instructions themselves are the list nodes.
521 auto next
= reinterpret_cast<TCA
>(m_state
.patches
[target
]);
522 auto here
= cb
.frontier();
524 // To avoid encoding 0x0 as the jump target. That would conflict with the use
525 // of nullptr as a sentinel return value from jmpTarget() and jccTarget().
526 // Consider switching those to use folly::Optional or something?
527 if (!next
) next
= kEndOfTargetChain
;
529 // This will never actually be executed as a jump to "next". It's just a
530 // pointer to the next jump instruction to retarget.
531 emitSmashableJump(cb
, next
, cc
);
532 m_state
.patches
[target
] = here
;
535 void patchJumps(CodeBlock
& cb
, CodegenState
& state
, Block
* block
) {
536 auto dest
= cb
.frontier();
537 auto jump
= reinterpret_cast<TCA
>(state
.patches
[block
]);
539 while (jump
&& jump
!= kEndOfTargetChain
) {
540 auto nextIfJmp
= jmpTarget(jump
);
541 auto nextIfJcc
= jccTarget(jump
);
543 // Exactly one of them must be non-nullptr
544 assert(!(nextIfJmp
&& nextIfJcc
));
545 assert(nextIfJmp
|| nextIfJcc
);
548 smashJmp(jump
, dest
);
551 smashJcc(jump
, dest
);
557 void emitFwdJmp(CodeBlock
& cb
, Block
* target
, CodegenState
& state
) {
558 always_assert(false);
561 //////////////////////////////////////////////////////////////////////
563 void CodeGenerator::recordHostCallSyncPoint(vixl::MacroAssembler
& as
,
565 auto stackOff
= m_curInst
->marker().spOff
;
566 auto pcOff
= m_curInst
->marker().bcOff
- m_curInst
->marker().func
->base();
567 m_mcg
->fixupMap().recordSyncPoint(tca
, pcOff
, stackOff
);
570 //////////////////////////////////////////////////////////////////////
572 void CodeGenerator::cgConjure(IRInstruction
* inst
) {
573 always_assert(false);
576 //////////////////////////////////////////////////////////////////////
578 void CodeGenerator::cgJmp(IRInstruction
* inst
) {
579 emitJumpToBlock(m_mcg
->code
.main(), inst
->taken(), CC_None
);
582 void CodeGenerator::cgDbgAssertRefCount(IRInstruction
* inst
) {
584 auto base
= x2a(srcLoc(0).reg());
586 m_as
. Ldr (rCount
.W(), base
[FAST_REFCOUNT_OFFSET
]);
587 m_as
. Tbnz (rCount
, UncountedBitPos
, &done
);
588 m_as
. Cmp (rCount
, RefCountMaxRealistic
);
589 m_as
. B (&done
, vixl::ls
);
594 void CodeGenerator::cgIncRef(IRInstruction
* inst
) {
595 SSATmp
* src
= inst
->src(0);
596 auto loc
= srcLoc(0);
597 Type type
= src
->type();
599 if (type
.notCounted()) return;
601 auto increfMaybeStatic
= [&] {
602 auto base
= x2a(loc
.reg(0));
603 auto rCount
= rAsm
.W();
604 m_as
. Ldr (rCount
, base
[FAST_REFCOUNT_OFFSET
]);
605 if (!type
.needsStaticBitCheck()) {
606 m_as
. Add (rCount
, rAsm
.W(), 1);
607 m_as
. Str (rCount
, base
[FAST_REFCOUNT_OFFSET
]);
609 m_as
. Cmp (rCount
, 0);
610 static_assert(UncountedValue
< 0 && StaticValue
< 0, "");
611 ifThen(m_as
, vixl::ge
, [&] {
612 m_as
.Add(rCount
, rCount
, 1);
613 m_as
.Str(rCount
, base
[FAST_REFCOUNT_OFFSET
]);
618 if (type
.isKnownDataType()) {
619 assert(IS_REFCOUNTED_TYPE(type
.toDataType()));
622 m_as
. Cmp (x2a(loc
.reg(1)).W(), KindOfRefCountThreshold
);
623 ifThen(m_as
, vixl::gt
, [&] { increfMaybeStatic(); });
627 void CodeGenerator::cgAssertType(IRInstruction
* inst
) {
628 auto const srcRegs
= srcLoc(0);
629 auto const dstRegs
= dstLoc(0);
631 PhysReg::Map
<PhysReg
> moves
;
632 if (dstRegs
.reg(0) != InvalidReg
)
633 moves
[dstRegs
.reg(0)] = srcRegs
.reg(0);
634 if (dstRegs
.reg(1) != InvalidReg
)
635 moves
[dstRegs
.reg(1)] = srcRegs
.reg(1);
637 auto howTo
= doRegMoves(moves
, rAsm
);
638 for (auto& how
: howTo
) {
639 if (how
.m_kind
== MoveInfo::Kind::Move
) {
640 m_as
. Mov (x2a(how
.m_dst
), x2a(how
.m_src
));
642 emitXorSwap(m_as
, x2a(how
.m_dst
), x2a(how
.m_src
));
647 //////////////////////////////////////////////////////////////////////
649 void CodeGenerator::emitDecRefStaticType(Type type
,
650 vixl::Register dataReg
) {
651 assert(type
.isKnownDataType());
652 assert(!dataReg
.Is(rAsm2
));
656 m_as
. Ldr (rAsm2
.W(), dataReg
[FAST_REFCOUNT_OFFSET
]);
658 if (type
.needsStaticBitCheck()) {
659 m_as
.Tbnz (rAsm2
, UncountedBitPos
, &allDone
);
662 m_as
. Sub (rAsm2
.W(), rAsm2
.W(), 1, vixl::SetFlags
);
663 m_as
. Str (rAsm2
.W(), dataReg
[FAST_REFCOUNT_OFFSET
]);
665 m_as
. B (&allDone
, vixl::ne
);
667 MCGenerator::getDtorCall(type
.toDataType()),
669 SyncOptions::kSyncPoint
,
670 argGroup().reg(dataReg
));
672 m_as
. bind (&allDone
);
675 void CodeGenerator::emitDecRefDynamicType(vixl::Register baseReg
,
677 // Make sure both temp registers are still available
678 assert(!baseReg
.Is(rAsm
));
679 assert(!baseReg
.Is(rAsm2
));
684 m_as
. Ldrb (rAsm
.W(), baseReg
[offset
+ TVOFF(m_type
)]);
685 m_as
. Cmp (rAsm
.W(), KindOfRefCountThreshold
);
686 m_as
. B (&allDone
, vixl::le
);
688 // Type is refcounted. Load the refcount.
689 m_as
. Ldr (rAsm
, baseReg
[offset
+ TVOFF(m_data
)]);
690 m_as
. Ldr (rAsm2
.W(), rAsm
[FAST_REFCOUNT_OFFSET
]);
692 // Is it static? Note that only the lower 32 bits of rAsm2 are valid right
693 // now, but tbnz is only looking at a single one of them, so this is OK.
694 m_as
. Tbnz (rAsm2
, UncountedBitPos
, &allDone
);
696 // Not static. Decrement and write back.
697 m_as
. Sub (rAsm2
.W(), rAsm2
.W(), 1, vixl::SetFlags
);
698 m_as
. Str (rAsm2
.W(), rAsm
[FAST_REFCOUNT_OFFSET
]);
700 // Did it go to zero?
701 m_as
. B (&allDone
, vixl::ne
);
703 // Went to zero. Have to destruct.
705 CppCall(tv_release_generic
),
707 SyncOptions::kSyncPoint
,
708 argGroup().addr(baseReg
, offset
));
710 m_as
. bind (&allDone
);
713 void CodeGenerator::emitDecRefMem(Type type
,
714 vixl::Register baseReg
,
716 if (type
.needsReg()) {
717 emitDecRefDynamicType(baseReg
, offset
);
718 } else if (type
.maybeCounted()) {
719 m_as
. Ldr (rAsm
, baseReg
[offset
+ TVOFF(m_data
)]);
720 emitDecRefStaticType(type
, rAsm
);
724 void CodeGenerator::cgDecRefStack(IRInstruction
* inst
) {
725 emitDecRefMem(inst
->typeParam(),
726 x2a(srcLoc(0).reg()),
727 cellsToBytes(inst
->extra
<DecRefStack
>()->offset
));
730 void CodeGenerator::cgDecRefLoc(IRInstruction
* inst
) {
731 emitDecRefMem(inst
->typeParam(),
732 x2a(srcLoc(0).reg()),
733 localOffset(inst
->extra
<DecRefLoc
>()->locId
));
736 void CodeGenerator::cgDecRefMem(IRInstruction
* inst
) {
737 emitDecRefMem(inst
->typeParam(),
738 x2a(srcLoc(0).reg()),
739 inst
->src(1)->intVal());
742 //////////////////////////////////////////////////////////////////////
743 // Arithmetic Instructions
745 void CodeGenerator::cgAddInt(IRInstruction
* inst
) {
746 auto dstReg
= dstLoc(0).reg();
747 auto srcRegL
= srcLoc(0).reg();
748 auto srcRegR
= srcLoc(1).reg();
750 if (srcRegR
!= InvalidReg
) {
751 m_as
. Add(x2a(dstReg
), x2a(srcRegL
), x2a(srcRegR
));
753 m_as
. Add(x2a(dstReg
), x2a(srcRegL
), inst
->src(1)->intVal());
757 void CodeGenerator::cgSubInt(IRInstruction
* inst
) {
758 auto dstReg
= dstLoc(0).reg();
759 auto srcRegL
= srcLoc(0).reg();
760 auto srcRegR
= srcLoc(1).reg();
762 if (srcRegR
!= InvalidReg
) {
763 m_as
. Sub(x2a(dstReg
), x2a(srcRegL
), x2a(srcRegR
));
765 m_as
. Sub(x2a(dstReg
), x2a(srcRegL
), inst
->src(1)->intVal());
769 void CodeGenerator::cgMulInt(IRInstruction
* inst
) {
770 auto dstReg
= dstLoc(0).reg();
771 auto srcRegL
= srcLoc(0).reg();
772 auto srcRegR
= srcLoc(1).reg();
774 m_as
. Mul(x2a(dstReg
), x2a(srcRegL
), x2a(srcRegR
));
777 //////////////////////////////////////////////////////////////////////
780 void CodeGenerator::cgAndInt(IRInstruction
* inst
) {
781 auto dstReg
= dstLoc(0).reg();
782 auto srcRegL
= srcLoc(0).reg();
783 auto srcRegR
= srcLoc(1).reg();
785 if (srcRegL
!= InvalidReg
) {
786 m_as
. And(x2a(dstReg
), x2a(srcRegL
), x2a(srcRegR
));
788 m_as
. And(x2a(dstReg
), x2a(srcRegL
), inst
->src(1)->intVal());
792 void CodeGenerator::cgOrInt(IRInstruction
* inst
) {
793 auto dstReg
= dstLoc(0).reg();
794 auto srcRegL
= srcLoc(0).reg();
795 auto srcRegR
= srcLoc(1).reg();
797 if (srcRegL
!= InvalidReg
) {
798 m_as
. Orr(x2a(dstReg
), x2a(srcRegL
), x2a(srcRegR
));
800 m_as
. Orr(x2a(dstReg
), x2a(srcRegL
), inst
->src(1)->intVal());
804 void CodeGenerator::cgXorInt(IRInstruction
* inst
) {
805 auto dstReg
= dstLoc(0).reg();
806 auto srcRegL
= srcLoc(0).reg();
807 auto srcRegR
= srcLoc(1).reg();
809 if (srcRegL
!= InvalidReg
) {
810 m_as
. Eor(x2a(dstReg
), x2a(srcRegL
), x2a(srcRegR
));
812 m_as
. Eor(x2a(dstReg
), x2a(srcRegL
), inst
->src(1)->intVal());
816 void CodeGenerator::cgShl(IRInstruction
* inst
) {
817 auto dstReg
= dstLoc(0).reg();
818 auto srcRegL
= srcLoc(0).reg();
819 auto srcRegR
= srcLoc(1).reg();
821 // TODO: t3870154 add shift-by-immediate support to vixl
822 m_as
. lslv(x2a(dstReg
), x2a(srcRegL
), x2a(srcRegR
));
825 void CodeGenerator::cgShr(IRInstruction
* inst
) {
826 auto dstReg
= dstLoc(0).reg();
827 auto srcRegL
= srcLoc(0).reg();
828 auto srcRegR
= srcLoc(1).reg();
830 // TODO: t3870154 add shift-by-immediate support to vixl
831 m_as
. asrv(x2a(dstReg
), x2a(srcRegL
), x2a(srcRegR
));
833 //////////////////////////////////////////////////////////////////////
834 // Comparison Operations
836 void CodeGenerator::emitCompareIntAndSet(IRInstruction
*inst
,
837 vixl::Condition cond
) {
838 auto dstReg
= dstLoc(0).reg();
839 emitCompareInt(inst
);
840 m_as
. Cset(x2a(dstReg
),cond
);
843 void CodeGenerator::emitCompareInt(IRInstruction
* inst
) {
844 auto srcRegL
= srcLoc(0).reg();
845 auto srcRegR
= srcLoc(1).reg();
847 if (srcRegR
!= InvalidReg
) {
848 m_as
. Cmp(x2a(srcRegL
), x2a(srcRegR
));
850 m_as
. Cmp(x2a(srcRegL
), inst
->src(1)->intVal());
854 void CodeGenerator::cgLtInt(IRInstruction
* inst
) {
855 emitCompareIntAndSet(inst
,vixl::Condition::lt
);
858 void CodeGenerator::cgGtInt(IRInstruction
* inst
) {
859 emitCompareIntAndSet(inst
,vixl::Condition::gt
);
863 void CodeGenerator::cgGteInt(IRInstruction
* inst
) {
864 emitCompareIntAndSet(inst
,vixl::Condition::ge
);
867 void CodeGenerator::cgLteInt(IRInstruction
* inst
) {
868 emitCompareIntAndSet(inst
,vixl::Condition::le
);
872 void CodeGenerator::cgEqInt(IRInstruction
* inst
) {
873 emitCompareIntAndSet(inst
,vixl::Condition::eq
);
876 void CodeGenerator::cgNeqInt(IRInstruction
* inst
) {
877 emitCompareIntAndSet(inst
,vixl::Condition::ne
);
880 //////////////////////////////////////////////////////////////////////
882 void CodeGenerator::cgShuffle(IRInstruction
* inst
) {
883 PhysReg::Map
<PhysReg
> moves
;
885 // Put required reg-reg moves in the map, and do spills at the same time.
886 for (auto i
= 0; i
< inst
->numSrcs(); ++i
) {
887 auto& rd
= inst
->extra
<Shuffle
>()->dests
[i
];
888 auto& rs
= srcLoc(i
);
890 if (rd
.numAllocated() == 0) continue;
892 for (auto j
= 0; j
< rd
.numAllocated(); ++j
) {
893 m_as
. Str (x2a(rs
.reg(j
)), vixl::MemOperand(vixl::sp
, rd
.offset(j
)));
895 } else if (!rs
.spilled()) {
896 if (rs
.reg(0) != InvalidReg
) moves
[rd
.reg(0)] = rs
.reg(0);
897 if (rs
.reg(1) != InvalidReg
) moves
[rd
.reg(1)] = rs
.reg(1);
902 auto howTo
= doRegMoves(moves
, rAsm
);
903 for (auto& how
: howTo
) {
904 if (how
.m_kind
== MoveInfo::Kind::Move
) {
905 emitRegGetsRegPlusImm(m_as
, x2a(how
.m_dst
), x2a(how
.m_src
), 0);
907 emitXorSwap(m_as
, x2a(how
.m_src
), x2a(how
.m_dst
));
911 // Now do reloads and reg<-imm.
912 for (auto i
= 0; i
< inst
->numSrcs(); ++i
) {
913 auto src
= inst
->src(i
);
914 auto& rd
= inst
->extra
<Shuffle
>()->dests
[i
];
915 auto& rs
= srcLoc(i
);
916 if (rd
.numAllocated() == 0) continue;
917 if (rd
.spilled()) continue;
919 for (auto j
= 0; j
< rd
.numAllocated(); ++j
) {
920 m_as
. Ldr (x2a(rd
.reg(i
)), vixl::MemOperand(vixl::sp
, rs
.offset(j
)));
924 if (rs
.numAllocated() == 0) {
925 assert(src
->inst()->op() == DefConst
);
926 m_as
. Mov (x2a(rd
.reg(0)), src
->rawVal());
928 if (rd
.numAllocated() == 2 && rs
.numAllocated() < 2) {
929 // Move src known type to register
930 m_as
. Mov (x2a(rd
.reg(1)), src
->type().toDataType());
936 //////////////////////////////////////////////////////////////////////
938 static void shuffleArgs(vixl::MacroAssembler
& a
,
941 assert_not_implemented(args
.numStackArgs() == 0);
943 PhysReg::Map
<PhysReg
> moves
;
944 PhysReg::Map
<ArgDesc
*> argDescs
;
946 for (size_t i
= 0; i
< args
.numRegArgs(); i
++) {
947 auto kind
= args
[i
].kind();
948 if (!(kind
== ArgDesc::Kind::Reg
||
949 kind
== ArgDesc::Kind::Addr
||
950 kind
== ArgDesc::Kind::TypeReg
)) {
953 auto dstReg
= args
[i
].dstReg();
954 auto srcReg
= args
[i
].srcReg();
955 if (dstReg
!= srcReg
) {
956 moves
[dstReg
] = srcReg
;
957 argDescs
[dstReg
] = &args
[i
];
959 if (call
.isIndirect() && dstReg
== call
.getReg()) {
960 // an indirect call uses an argument register for the func ptr.
961 // Use rAsm2 instead and update the CppCall
962 moves
[rAsm2
] = call
.getReg();
963 call
.updateCallIndirect(rAsm2
);
967 auto const howTo
= doRegMoves(moves
, rAsm
);
969 for (auto& how
: howTo
) {
970 auto srcReg
= x2a(how
.m_src
);
971 auto dstReg
= x2a(how
.m_dst
);
972 if (how
.m_kind
== MoveInfo::Kind::Move
) {
973 auto* argDesc
= argDescs
[how
.m_dst
];
975 auto kind
= argDesc
->kind();
976 if (kind
== ArgDesc::Kind::Addr
) {
977 emitRegGetsRegPlusImm(a
, dstReg
, srcReg
, argDesc
->disp().l());
979 if (argDesc
->isZeroExtend()) {
980 // "Unsigned eXTend Byte". The dest reg is a 32-bit reg but this
981 // zeroes the top 32 bits, so the intended effect is achieved.
982 a
.Uxtb (dstReg
.W(), srcReg
.W());
984 a
.Mov (dstReg
, srcReg
);
987 if (kind
!= ArgDesc::Kind::TypeReg
) {
991 a
. Mov (dstReg
, srcReg
);
994 emitXorSwap(a
, dstReg
, srcReg
);
998 for (size_t i
= 0; i
< args
.numRegArgs(); ++i
) {
999 if (!args
[i
].done()) {
1000 auto kind
= args
[i
].kind();
1001 auto dstReg
= x2a(args
[i
].dstReg());
1002 if (kind
== ArgDesc::Kind::Imm
) {
1003 a
. Mov (dstReg
, args
[i
].imm().q());
1004 } else if (kind
== ArgDesc::Kind::Reg
|| kind
== ArgDesc::Kind::TypeReg
) {
1005 // Should have already been done
1013 void CodeGenerator::cgCallNative(vixl::MacroAssembler
& as
,
1014 IRInstruction
* inst
) {
1015 using namespace NativeCalls
;
1017 Opcode opc
= inst
->op();
1018 always_assert(CallMap::hasInfo(opc
));
1020 auto const& info
= CallMap::info(opc
);
1021 ArgGroup argGroup
= info
.toArgGroup(m_state
.regs
, inst
);
1023 auto call
= [&]() -> CppCall
{
1024 switch (info
.func
.type
) {
1025 case FuncType::Call
:
1026 return CppCall(info
.func
.call
);
1028 return CppCall(inst
->src(info
.func
.srcIdx
)->tcaVal());
1033 auto const dest
= [&]() -> CallDest
{
1034 switch (info
.dest
) {
1035 case DestType::None
: return kVoidDest
;
1036 case DestType::TV
: return callDestTV(inst
);
1037 case DestType::SSA
: return callDest(inst
);
1038 case DestType::SSA2
: return callDest2(inst
);
1043 cgCallHelper(as
, call
, dest
, info
.sync
, argGroup
);
1046 void CodeGenerator::cgCallHelper(vixl::MacroAssembler
& a
,
1048 const CallDest
& dstInfo
,
1052 assert(m_curInst
->isNative());
1054 auto dstReg0
= dstInfo
.reg0
;
1055 auto dstReg1
= dstInfo
.reg1
;
1058 toSave
.forEach([](PhysReg r
) { assert(r
.isGP()); });
1061 toSave
= toSave
& kCallerSaved
;
1062 assert((toSave
& RegSet().add(dstReg0
).add(dstReg1
)).empty());
1064 // Use vixl's handy helper to push caller-save regs. It uses ldp/stp when
1066 CPURegList
pushedRegs(vixl::CPURegister::kRegister
, vixl::kXRegSize
, 0);
1067 toSave
.forEach([&](PhysReg r
) { pushedRegs
.Combine(r
); });
1069 // The vixl helper requires you to pass it an even number of registers. If we
1070 // have an odd number of regs to save, remove one from the list we pass, and
1071 // save it ourselves.
1072 folly::Optional
<vixl::CPURegister
> maybeOddOne
;
1073 if (pushedRegs
.Count() % 2 == 1) {
1074 maybeOddOne
= pushedRegs
.PopHighestIndex();
1076 a
. PushCPURegList(pushedRegs
);
1078 // We're only storing a single reg, but the stack pointer must always be
1079 // 16-byte aligned. This instruction subtracts 16 from the stack pointer,
1080 // then writes the value.
1081 a
. Str (maybeOddOne
.value(), MemOperand(vixl::sp
, -16, vixl::PreIndex
));
1086 // Read the value, then add 16 to the stack pointer.
1087 a
.Ldr (maybeOddOne
.value(), MemOperand(vixl::sp
, 16, vixl::PostIndex
));
1089 a
. PopCPURegList(pushedRegs
);
1092 for (size_t i
= 0; i
< args
.numRegArgs(); i
++) {
1093 args
[i
].setDstReg(PhysReg
{argReg(i
)});
1095 shuffleArgs(a
, args
, call
);
1097 auto syncPoint
= emitCall(a
, call
);
1099 if (RuntimeOption::HHProfServerEnabled
|| sync
!= SyncOptions::kNoSyncPoint
) {
1100 recordHostCallSyncPoint(a
, syncPoint
);
1103 auto armDst0
= x2a(dstReg0
);
1104 DEBUG_ONLY
auto armDst1
= x2a(dstReg1
);
1106 switch (dstInfo
.type
) {
1107 case DestType::TV
: not_implemented();
1109 assert(!armDst1
.IsValid());
1110 if (armDst0
.IsValid() && !armDst0
.Is(vixl::x0
)) a
.Mov(armDst0
, vixl::x0
);
1112 case DestType::SSA2
: not_implemented();
1113 case DestType::None
:
1114 assert(!armDst0
.IsValid() && !armDst1
.IsValid());
1119 void CodeGenerator::cgCallHelper(vixl::MacroAssembler
& a
,
1121 const CallDest
& dstInfo
,
1124 cgCallHelper(a
, call
, dstInfo
, sync
, args
, m_state
.liveRegs
[m_curInst
]);
1128 * XXX copypasta but has to be in the class because of curPhysLoc and
1129 * changing that would make callsites real messy
1132 CallDest
CodeGenerator::callDest(PhysReg reg0
,
1133 PhysReg reg1
/* = InvalidReg */) const {
1134 return { DestType::SSA
, reg0
, reg1
};
1137 CallDest
CodeGenerator::callDest(const IRInstruction
* inst
) const {
1138 if (!inst
->numDsts()) return kVoidDest
;
1139 auto loc
= dstLoc(0);
1140 return { DestType::SSA
, loc
.reg(0), loc
.reg(1) };
1143 CallDest
CodeGenerator::callDestTV(const IRInstruction
* inst
) const {
1144 if (!inst
->numDsts()) return kVoidDest
;
1145 auto loc
= dstLoc(0);
1146 return { DestType::TV
, loc
.reg(0), loc
.reg(1) };
1149 CallDest
CodeGenerator::callDest2(const IRInstruction
* inst
) const {
1150 if (!inst
->numDsts()) return kVoidDest
;
1151 auto loc
= dstLoc(0);
1152 return { DestType::SSA2
, loc
.reg(0), loc
.reg(1) };
1155 //////////////////////////////////////////////////////////////////////
1157 static vixl::Register
enregister(vixl::MacroAssembler
& a
,
1158 vixl::MemOperand memRef
,
1159 vixl::Register scratch
) {
1160 a
. Ldr (scratch
, memRef
);
1164 static vixl::Register
enregister(vixl::MacroAssembler
& a
,
1166 vixl::Register scratch
) {
1170 template<class Loc
, class JmpFn
>
1171 void CodeGenerator::emitTypeTest(Type type
, vixl::Register typeReg
, Loc dataSrc
,
1173 assert(!(type
<= Type::Cls
));
1174 assert(typeReg
.Is32Bits());
1176 if (type
.equals(Type::Gen
)) {
1181 if (type
<= Type::Str
) {
1182 // Note: ARM can actually do better here; it has a fused test-and-branch
1183 // instruction. The way this code is factored makes it difficult to use,
1184 // though; the jump instruction will be written by some other code.
1185 m_as
. Tst (typeReg
, KindOfStringBit
);
1187 } else if (type
.equals(Type::UncountedInit
)) {
1188 m_as
. Tst (typeReg
, KindOfUncountedInitBit
);
1190 } else if (type
.equals(Type::Uncounted
)) {
1191 m_as
. Cmp (typeReg
, KindOfRefCountThreshold
);
1193 } else if (type
.equals(Type::Cell
)) {
1194 m_as
. Cmp (typeReg
, KindOfRef
);
1197 assert(type
.isKnownDataType());
1198 DataType dataType
= type
.toDataType();
1199 assert(dataType
== KindOfRef
||
1200 (dataType
>= KindOfUninit
&& dataType
<= KindOfResource
));
1201 m_as
. Cmp (typeReg
, dataType
);
1205 if (type
< Type::Obj
) {
1206 assert(type
.getClass()->attrs() & AttrFinal
);
1207 auto dataReg
= enregister(m_as
, dataSrc
, rAsm
);
1208 m_as
. Ldr (rAsm
, dataReg
[ObjectData::getVMClassOffset()]);
1209 m_as
. Cmp (rAsm
, reinterpret_cast<int64_t>(type
.getClass()));
1211 } else if (type
< Type::Res
) {
1212 CG_PUNT(TypeTest
-on
-Resource
);
1213 } else if (type
<= Type::Arr
&& type
.hasArrayKind()) {
1214 auto dataReg
= enregister(m_as
, dataSrc
, rAsm
);
1215 m_as
. Ldrb (rAsm
.W(), dataReg
[ArrayData::offsetofKind()]);
1216 m_as
. Cmp (rAsm
.W(), type
.getArrayKind());
1221 void CodeGenerator::cgGuardLoc(IRInstruction
* inst
) {
1222 auto const rFP
= x2a(srcLoc(0).reg());
1223 auto const baseOff
= localOffset(inst
->extra
<GuardLoc
>()->locId
);
1224 m_as
. Ldrb (rAsm
.W(), rFP
[baseOff
+ TVOFF(m_type
)]);
1228 rFP
[baseOff
+ TVOFF(m_data
)],
1229 [&] (ConditionCode cc
) {
1230 auto const destSK
= SrcKey(curFunc(), m_unit
.bcOff());
1231 auto const destSR
= m_mcg
->tx().getSrcRec(destSK
);
1232 destSR
->emitFallbackJump(this->m_mainCode
, ccNegate(cc
));
1236 void CodeGenerator::cgGuardStk(IRInstruction
* inst
) {
1237 auto const rSP
= x2a(srcLoc(0).reg());
1238 auto const baseOff
= cellsToBytes(inst
->extra
<GuardStk
>()->offset
);
1239 m_as
. Ldrb (rAsm
.W(), rSP
[baseOff
+ TVOFF(m_type
)]);
1243 rSP
[baseOff
+ TVOFF(m_data
)],
1244 [&] (ConditionCode cc
) {
1245 auto const destSK
= SrcKey(curFunc(), m_unit
.bcOff());
1246 auto const destSR
= m_mcg
->tx().getSrcRec(destSK
);
1247 destSR
->emitFallbackJump(this->m_mainCode
, ccNegate(cc
));
1251 void CodeGenerator::cgCheckStk(IRInstruction
* inst
) {
1252 auto const rSP
= x2a(srcLoc(0).reg());
1253 auto const baseOff
= cellsToBytes(inst
->extra
<CheckStk
>()->offset
);
1254 m_as
. Ldrb (rAsm
.W(), rSP
[baseOff
+ TVOFF(m_type
)]);
1258 rSP
[baseOff
+ TVOFF(m_data
)],
1259 [&] (ConditionCode cc
) {
1260 emitJumpToBlock(m_mcg
->code
.main(), inst
->taken(), ccNegate(cc
));
1265 void CodeGenerator::cgCheckType(IRInstruction
* inst
) {
1266 auto const src
= inst
->src(0);
1267 Type srcType
= src
->type();
1268 auto const rVal
= x2a(srcLoc(0).reg(0));
1269 auto const rType
= x2a(srcLoc(0).reg(1));
1272 auto const valDst
= x2a(dstLoc(0).reg(0));
1273 auto const typeDst
= x2a(dstLoc(0).reg(1));
1274 // TODO: #3626251: XLS: Let Uses say whether a constant is
1275 // allowed, and if not, assign a register.
1276 if (valDst
.IsValid()) {
1277 if (rVal
.IsValid()) {
1278 if (!valDst
.Is(rVal
)) m_as
.Mov(valDst
, rVal
);
1280 if (src
->isConst()) m_as
.Mov(valDst
, src
->rawVal());
1283 if (typeDst
.IsValid()) {
1284 if (rType
.IsValid()) {
1285 if (!typeDst
.Is(rType
)) m_as
.Mov(typeDst
, rType
);
1287 m_as
.Mov(typeDst
, srcType
.toDataType());
1292 auto doJcc
= [&] (ConditionCode cc
) {
1293 emitJumpToBlock(m_mcg
->code
.main(), inst
->taken(), ccNegate(cc
));
1296 Type typeParam
= inst
->typeParam();
1297 if (src
->isA(typeParam
) ||
1298 // Boxed types are checked lazily, so there's nothing to be done here.
1299 (srcType
.isBoxed() && typeParam
.isBoxed())) {
1303 if (srcType
.not(typeParam
)) {
1304 emitJumpToBlock(m_mcg
->code
.main(), inst
->taken(), CC_None
);
1308 if (rType
.IsValid()) {
1309 emitTypeTest(typeParam
, rType
.W(), rVal
, doJcc
);
1310 } else if (typeParam
<= Type::Uncounted
&&
1311 ((srcType
== Type::Str
&& typeParam
.maybe(Type::StaticStr
)) ||
1312 (srcType
== Type::Arr
&& typeParam
.maybe(Type::StaticArr
)))) {
1313 // We carry Str and Arr operands around without a type register,
1314 // even though they're union types. The static and non-static
1315 // subtypes are distinguised by the refcount field.
1316 assert(rVal
.IsValid());
1317 m_as
. Ldr (rAsm
.W(), rVal
[FAST_REFCOUNT_OFFSET
]);
1318 m_as
. Cmp (rAsm
, 0);
1324 return folly::format("Bad src: {} and dst: {} types in '{}'",
1325 srcType
, typeParam
, *inst
).str();
1331 void CodeGenerator::cgSideExitGuardStk(IRInstruction
* inst
) {
1332 auto const sp
= x2a(srcLoc(0).reg());
1333 auto const extra
= inst
->extra
<SideExitGuardStk
>();
1335 m_as
. Ldrb (rAsm
.W(), sp
[cellsToBytes(extra
->checkedSlot
) + TVOFF(m_type
)]);
1339 sp
[cellsToBytes(extra
->checkedSlot
) + TVOFF(m_data
)],
1340 [&] (ConditionCode cc
) {
1341 auto const sk
= SrcKey(curFunc(), extra
->taken
);
1342 emitBindSideExit(this->m_mainCode
, this->m_stubsCode
, sk
, ccNegate(cc
));
1347 void CodeGenerator::cgGuardRefs(IRInstruction
* inst
) {
1348 assert(inst
->numSrcs() == 5);
1350 DEBUG_ONLY SSATmp
* nParamsTmp
= inst
->src(1);
1351 DEBUG_ONLY SSATmp
* firstBitNumTmp
= inst
->src(2);
1352 DEBUG_ONLY SSATmp
* mask64Tmp
= inst
->src(3);
1353 DEBUG_ONLY SSATmp
* vals64Tmp
= inst
->src(4);
1355 auto funcPtrLoc
= srcLoc(0);
1356 auto nParamsLoc
= srcLoc(1);
1357 auto mask64Loc
= srcLoc(3);
1358 auto vals64Loc
= srcLoc(4);
1360 // Get values in place
1361 auto funcPtrReg
= x2a(funcPtrLoc
.reg());
1362 assert(funcPtrReg
.IsValid());
1364 auto nParamsReg
= x2a(nParamsLoc
.reg());
1365 assert(nParamsReg
.IsValid() || nParamsTmp
->isConst());
1367 auto firstBitNum
= static_cast<uint32_t>(firstBitNumTmp
->intVal());
1368 auto mask64Reg
= x2a(mask64Loc
.reg());
1369 uint64_t mask64
= mask64Tmp
->intVal();
1370 assert(mask64Reg
.IsValid() || mask64
== uint32_t(mask64
));
1373 auto vals64Reg
= x2a(vals64Loc
.reg());
1374 uint64_t vals64
= vals64Tmp
->intVal();
1375 assert(vals64Reg
.IsValid() || vals64
== uint32_t(vals64
));
1376 assert((vals64
& mask64
) == vals64
);
1378 auto const destSK
= SrcKey(curFunc(), m_unit
.bcOff());
1379 auto const destSR
= m_mcg
->tx().getSrcRec(destSK
);
1381 auto thenBody
= [&] {
1382 auto bitsOff
= sizeof(uint64_t) * (firstBitNum
/ 64);
1384 auto bitsPtrReg
= rAsm
;
1386 if (firstBitNum
== 0) {
1387 bitsOff
= Func::refBitValOff();
1388 bitsPtrReg
= funcPtrReg
;
1390 m_as
. Ldr (bitsPtrReg
, funcPtrReg
[Func::sharedOff()]);
1391 bitsOff
-= sizeof(uint64_t);
1394 // Don't need the bits pointer after this point
1395 auto bitsReg
= rAsm
;
1397 m_as
. Ldr (bitsReg
, bitsPtrReg
[bitsOff
]);
1399 // Mask the bits. There are restrictions on what can be encoded as an
1400 // immediate in ARM's logical instructions, and if they're not met,
1401 // the assembler will compensate using ip0 or ip1 as tmps.
1402 if (mask64Reg
.IsValid()) {
1403 m_as
. And (bitsReg
, bitsReg
, mask64Reg
);
1405 m_as
. And (bitsReg
, bitsReg
, mask64
);
1408 // Now do the compare. There are also restrictions on immediates in
1409 // arithmetic instructions (of which Cmp is one; it's just a subtract that
1410 // sets flags), so same deal as with the mask immediate above.
1411 if (vals64Reg
.IsValid()) {
1412 m_as
. Cmp (bitsReg
, vals64Reg
);
1414 m_as
. Cmp (bitsReg
, vals64
);
1416 destSR
->emitFallbackJump(m_mainCode
, cond
);
1419 if (firstBitNum
== 0) {
1420 assert(!nParamsReg
.IsValid());
1421 // This is the first 64 bits. No need to check
1425 assert(nParamsReg
.IsValid());
1426 // Check number of args...
1427 m_as
. Cmp (nParamsReg
, firstBitNum
);
1429 if (vals64
!= 0 && vals64
!= mask64
) {
1430 // If we're beyond nParams, then either all params
1431 // are refs, or all params are non-refs, so if vals64
1432 // isn't 0 and isnt mask64, there's no possibility of
1434 destSR
->emitFallbackJump(m_mainCode
, CC_LE
);
1437 ifThenElse(m_as
, vixl::gt
, thenBody
, /* else */ [&] {
1438 // If not special builtin...
1439 m_as
. Ldr (rAsm
, funcPtrReg
[Func::attrsOff()]);
1440 m_as
. Tst (rAsm
, AttrVariadicByRef
);
1441 destSR
->emitFallbackJump(m_mainCode
, vals64
? CC_Z
: CC_NZ
);
1447 //////////////////////////////////////////////////////////////////////
1449 void CodeGenerator::cgSyncABIRegs(IRInstruction
* inst
) {
1450 emitRegGetsRegPlusImm(m_as
, rVmFp
, x2a(srcLoc(0).reg()), 0);
1451 emitRegGetsRegPlusImm(m_as
, rVmSp
, x2a(srcLoc(1).reg()), 0);
1454 void CodeGenerator::cgReqBindJmp(IRInstruction
* inst
) {
1458 SrcKey(curFunc(), inst
->extra
<ReqBindJmp
>()->offset
)
1462 void CodeGenerator::cgReqRetranslate(IRInstruction
* inst
) {
1463 assert(m_unit
.bcOff() == inst
->marker().bcOff
);
1464 auto const destSK
= SrcKey(curFunc(), m_unit
.bcOff());
1465 auto const destSR
= m_mcg
->tx().getSrcRec(destSK
);
1466 destSR
->emitFallbackJump(m_mainCode
);
1469 void CodeGenerator::cgSpillFrame(IRInstruction
* inst
) {
1470 auto const func
= inst
->src(2);
1471 auto const objOrCls
= inst
->src(3);
1472 auto const invName
= inst
->extra
<SpillFrame
>()->invName
;
1473 auto const nArgs
= inst
->extra
<SpillFrame
>()->numArgs
;
1475 auto spReg
= x2a(srcLoc(0).reg());
1476 auto fpReg
= x2a(srcLoc(1).reg());
1477 auto funcLoc
= srcLoc(2);
1478 auto objClsReg
= x2a(srcLoc(3).reg());
1479 auto spOff
= -kNumActRecCells
* sizeof(Cell
);
1482 m_as
. Str (fpReg
, spReg
[spOff
+ AROFF(m_savedRbp
)]);
1484 // Num args. Careful here: nArgs is 32 bits and the high bit may be set. Mov's
1485 // immediate argument is intptr_t, and the implicit int32->intptr conversion
1486 // will sign-extend, which isn't what we want.
1487 m_as
. Mov (rAsm
.W(), (uint32_t)nArgs
);
1488 m_as
. Str (rAsm
.W(), spReg
[spOff
+ AROFF(m_numArgsAndGenCtorFlags
)]);
1492 auto bits
= reinterpret_cast<uintptr_t>(invName
) | ActRec::kInvNameBit
;
1493 m_as
. Mov (rAsm
, bits
);
1494 m_as
. Str (rAsm
, spReg
[spOff
+ AROFF(m_invName
)]);
1496 m_as
. Str (vixl::xzr
, spReg
[spOff
+ AROFF(m_invName
)]);
1499 // Func and this/class are slightly tricky. The func may be a tuple of a Func*
1502 if (objOrCls
->isA(Type::Cls
)) {
1503 if (objOrCls
->isConst()) {
1504 m_as
.Mov (rAsm
, uintptr_t(objOrCls
->clsVal()) | 1);
1505 m_as
.Str (rAsm
, spReg
[spOff
+ AROFF(m_this
)]);
1507 m_as
.Orr (rAsm
, objClsReg
, 1);
1508 m_as
.Str (rAsm
, spReg
[spOff
+ AROFF(m_this
)]);
1510 } else if (objOrCls
->isA(Type::Obj
) || objOrCls
->isA(Type::Ctx
)) {
1511 m_as
. Str (objClsReg
, spReg
[spOff
+ AROFF(m_this
)]);
1513 assert(objOrCls
->isA(Type::InitNull
));
1514 m_as
.Str (vixl::xzr
, spReg
[spOff
+ AROFF(m_this
)]);
1517 // Now set func, and possibly this/cls
1518 if (func
->isA(Type::Null
)) {
1520 assert(func
->isConst());
1521 } else if (func
->isConst()) {
1522 m_as
. Mov (rAsm
, func
->funcVal());
1523 m_as
. Str (rAsm
, spReg
[spOff
+ AROFF(m_func
)]);
1525 auto reg0
= x2a(funcLoc
.reg(0));
1526 m_as
. Str (reg0
, spReg
[spOff
+ AROFF(m_func
)]);
1529 // Adjust stack pointer
1530 emitRegGetsRegPlusImm(m_as
, x2a(dstLoc(0).reg()), spReg
, spOff
);
1533 //////////////////////////////////////////////////////////////////////
1535 void CodeGenerator::cgCallBuiltin(IRInstruction
* inst
) {
1536 auto func
= inst
->src(0)->funcVal();
1537 auto args
= inst
->srcs().subpiece(2);
1538 auto numArgs
= args
.size();
1540 DataType funcReturnType
= func
->returnType();
1541 int returnOffset
= MISOFF(tvBuiltinReturn
);
1543 if (FixupMap::eagerRecord(func
)) {
1544 // Save VM registers
1545 auto const* pc
= curFunc()->unit()->entry() + m_curInst
->marker().bcOff
;
1546 m_as
.Str (rVmFp
, rGContextReg
[offsetof(ExecutionContext
, m_fp
)]);
1547 m_as
.Str (rVmSp
, rGContextReg
[offsetof(ExecutionContext
, m_stack
) +
1548 Stack::topOfStackOffset()]);
1549 m_as
.Mov (rAsm
, pc
);
1550 m_as
.Str (rAsm
, rGContextReg
[offsetof(ExecutionContext
, m_pc
)]);
1553 // The stack pointer currently points to the MInstrState we need to use.
1555 m_as
. Mov (rAsm
, vixl::sp
);
1557 auto callArgs
= argGroup();
1558 if (isCppByRef(funcReturnType
)) {
1559 if (isSmartPtrRef(funcReturnType
)) {
1560 // first arg is pointer to storage for the return value
1561 returnOffset
+= TVOFF(m_data
);
1563 callArgs
.addr(misReg
, returnOffset
);
1566 for (int i
= 0; i
< numArgs
; ++i
) {
1567 auto const& pi
= func
->params()[i
];
1568 if (TVOFF(m_data
) && isSmartPtrRef(pi
.builtinType())) {
1569 callArgs
.addr(srcLoc(i
+ 2).reg(), TVOFF(m_data
));
1571 callArgs
.ssa(i
+ 2);
1577 auto dloc
= dstLoc(0);
1578 auto dstReg
= x2a(dloc
.reg(0));
1579 auto dstTypeReg
= x2a(dloc
.reg(1));
1582 CppCall((TCA
)func
->nativeFuncPtr()),
1583 isCppByRef(funcReturnType
) ? kVoidDest
: callDest(dstReg
),
1584 SyncOptions::kSyncPoint
,
1587 auto returnType
= inst
->typeParam();
1588 if (!dstReg
.IsValid() || returnType
.isSimpleType()) {
1592 if (returnType
.isReferenceType()) {
1593 assert(isCppByRef(funcReturnType
) && isSmartPtrRef(funcReturnType
));
1594 vixl::Label notNullptr
;
1596 m_as
. Ldr (dstReg
, misReg
[returnOffset
+ TVOFF(m_data
)]);
1597 m_as
. Cbnz (dstReg
, ¬Nullptr
);
1598 m_as
. Mov (dstTypeReg
, KindOfNull
);
1600 m_as
. bind (¬Nullptr
);
1601 m_as
. Mov (dstTypeReg
, returnType
.toDataType());
1606 if (returnType
<= Type::Cell
|| returnType
<= Type::BoxedCell
) {
1607 assert(isCppByRef(funcReturnType
) && !isSmartPtrRef(funcReturnType
));
1608 vixl::Label notUninit
;
1610 m_as
. Ldrb (dstTypeReg
.W(), misReg
[returnOffset
+ TVOFF(m_type
)]);
1611 m_as
. Cbnz (dstTypeReg
, ¬Uninit
);
1612 m_as
. Mov (dstTypeReg
, KindOfNull
);
1614 m_as
. bind (¬Uninit
);
1615 m_as
. Ldr (dstReg
, misReg
[returnOffset
+ TVOFF(m_data
)]);
1620 always_assert(false);
1623 void CodeGenerator::cgCall(IRInstruction
* inst
) {
1624 auto spReg
= x2a(srcLoc(0).reg());
1625 auto* returnBcOffset
= inst
->src(1);
1626 auto* func
= inst
->src(2);
1627 SrcRange args
= inst
->srcs().subpiece(3);
1628 int32_t numArgs
= args
.size();
1630 int64_t adjustment
= cellsToBytes((int64_t)-numArgs
);
1631 for (int32_t i
= 0; i
< numArgs
; ++i
) {
1632 emitStore(spReg
, cellsToBytes(-(i
+ 1)), args
[i
], srcLoc(i
+ 3));
1635 m_as
. Mov (rAsm
.W(), returnBcOffset
->intVal());
1636 m_as
. Str (rAsm
.W(), spReg
[AROFF(m_soff
)]);
1637 emitRegGetsRegPlusImm(m_as
, spReg
, spReg
, adjustment
);
1639 assert(m_curInst
->marker().valid());
1640 SrcKey srcKey
= SrcKey(m_curInst
->marker().func
, m_curInst
->marker().bcOff
);
1641 bool isImmutable
= func
->isConst() && !func
->isA(Type::Null
);
1642 const Func
* funcd
= isImmutable
? func
->funcVal() : nullptr;
1643 int32_t adjust
= emitBindCall(mcg
->code
.main(), mcg
->code
.stubs(),
1644 srcKey
, funcd
, numArgs
);
1646 emitRegGetsRegPlusImm(m_as
, rVmSp
, rVmSp
, adjust
);
1649 //////////////////////////////////////////////////////////////////////
1651 void CodeGenerator::cgBeginCatch(IRInstruction
* inst
) {
1655 void CodeGenerator::cgEndCatch(IRInstruction
* inst
) {
1659 //////////////////////////////////////////////////////////////////////
1661 void CodeGenerator::emitLoadTypedValue(PhysLoc dst
,
1662 vixl::Register base
,
1665 auto valueDstReg
= x2a(dst
.reg(0));
1666 auto typeDstReg
= x2a(dst
.reg(1));
1668 if (label
) not_implemented();
1670 if (valueDstReg
.IsFPRegister()) {
1674 // Avoid clobbering the base reg if we'll need it later
1675 if (base
.Is(typeDstReg
) && valueDstReg
.IsValid()) {
1676 m_as
. Mov (rAsm
, base
);
1680 if (typeDstReg
.IsValid()) {
1681 m_as
. Ldrb (typeDstReg
.W(), base
[offset
+ TVOFF(m_type
)]);
1684 if (valueDstReg
.IsValid()) {
1685 m_as
. Ldr (valueDstReg
, base
[offset
+ TVOFF(m_data
)]);
1689 void CodeGenerator::emitStoreTypedValue(vixl::Register base
,
1692 assert(src
.numWords() == 2);
1693 auto reg0
= x2a(src
.reg(0));
1694 auto reg1
= x2a(src
.reg(1));
1695 m_as
. Str (reg0
, base
[offset
+ TVOFF(m_data
)]);
1696 m_as
. Strb (reg1
.W(), base
[offset
+ TVOFF(m_type
)]);
1699 void CodeGenerator::emitLoad(Type type
, PhysLoc dstLoc
,
1700 vixl::Register base
,
1702 Block
* label
/* = nullptr */) {
1703 if (type
.needsReg()) {
1704 return emitLoadTypedValue(dstLoc
, base
, offset
, label
);
1709 if (type
<= Type::Null
) return;
1711 auto dstReg
= x2a(dstLoc
.reg());
1712 if (!dstReg
.IsValid()) return;
1714 m_as
. Ldr (dstReg
, base
[offset
+ TVOFF(m_data
)]);
1717 void CodeGenerator::emitStore(vixl::Register base
,
1719 SSATmp
* src
, PhysLoc srcLoc
,
1720 bool genStoreType
/* = true */) {
1721 auto type
= src
->type();
1722 if (type
.needsReg()) {
1723 return emitStoreTypedValue(base
, offset
, srcLoc
);
1726 auto dt
= type
.toDataType();
1727 if (dt
== KindOfUninit
) {
1728 static_assert(KindOfUninit
== 0, "zero register hack");
1729 m_as
. Strb (vixl::wzr
, base
[offset
+ TVOFF(m_type
)]);
1731 m_as
. Mov (rAsm
, dt
);
1732 m_as
. Strb (rAsm
.W(), base
[offset
+ TVOFF(m_type
)]);
1735 if (type
<= Type::Null
) {
1738 if (src
->isConst()) {
1740 if (type
<= (Type::Bool
| Type::Int
| Type::Dbl
|
1741 Type::Arr
| Type::StaticStr
| Type::Cls
)) {
1742 val
= src
->rawVal();
1746 m_as
. Mov (rAsm
, val
);
1747 m_as
. Str (rAsm
, base
[offset
+ TVOFF(m_data
)]);
1749 auto reg
= x2a(srcLoc
.reg());
1750 if (src
->isA(Type::Bool
)) {
1751 m_as
. Uxtb (reg
.W(), reg
.W());
1753 m_as
. Str (x2a(srcLoc
.reg()), base
[offset
+ TVOFF(m_data
)]);
1757 void CodeGenerator::cgLdLoc(IRInstruction
* inst
) {
1758 auto baseReg
= x2a(srcLoc(0).reg());
1759 auto offset
= localOffset(inst
->extra
<LdLoc
>()->locId
);
1760 emitLoad(inst
->dst()->type(), dstLoc(0), baseReg
, offset
);
1763 void CodeGenerator::cgStLoc(IRInstruction
* inst
) {
1764 auto baseReg
= x2a(srcLoc(0).reg());
1765 auto offset
= localOffset(inst
->extra
<StLoc
>()->locId
);
1766 emitStore(baseReg
, offset
, inst
->src(1), srcLoc(1), true /* store type */);
1769 void CodeGenerator::cgLdStack(IRInstruction
* inst
) {
1770 assert(inst
->taken() == nullptr);
1771 auto srcReg
= x2a(srcLoc(0).reg());
1772 auto offset
= cellsToBytes(inst
->extra
<LdStack
>()->offset
);
1773 emitLoad(inst
->dst()->type(), dstLoc(0), srcReg
, offset
);
1776 void CodeGenerator::emitLdRaw(IRInstruction
* inst
, size_t extraOff
) {
1777 auto destReg
= x2a(dstLoc(0).reg());
1778 auto offset
= inst
->extra
<RawMemData
>()->info().offset
;
1779 auto src
= x2a(srcLoc(0).reg())[offset
+ extraOff
];
1781 switch (inst
->extra
<RawMemData
>()->info().size
) {
1782 case sz::byte
: m_as
. Ldrb (destReg
.W(), src
); break;
1783 case sz::dword
: m_as
. Ldr (destReg
.W(), src
); break;
1784 case sz::qword
: m_as
. Ldr (destReg
, src
); break;
1785 default: not_implemented();
1789 void CodeGenerator::cgLdRaw(IRInstruction
* inst
) {
1793 void CodeGenerator::cgLdContArRaw(IRInstruction
* inst
) {
1794 emitLdRaw(inst
, -c_Continuation::getArOffset());
1797 void CodeGenerator::cgLdARFuncPtr(IRInstruction
* inst
) {
1798 auto dstReg
= x2a(dstLoc(0).reg());
1799 auto baseReg
= x2a(srcLoc(0).reg());
1800 auto offset
= inst
->src(1)->intVal();
1801 m_as
. Ldr (dstReg
, baseReg
[offset
+ AROFF(m_func
)]);
1804 void CodeGenerator::cgLdFuncCached(IRInstruction
* inst
) {
1805 auto dstReg
= x2a(dstLoc(0).reg());
1806 auto const name
= inst
->extra
<LdFuncCachedData
>()->name
;
1807 auto const ch
= Unit::GetNamedEntity(name
)->getFuncHandle();
1808 vixl::Label noLookup
;
1810 if (!dstReg
.IsValid()) {
1811 m_as
. Ldr (rAsm
, rVmTl
[ch
]);
1814 m_as
. Ldr (dstReg
, rVmTl
[ch
]);
1816 m_as
. Cbnz (dstReg
, &noLookup
);
1818 const Func
* (*const func
)(const StringData
*) = lookupUnknownFunc
;
1823 SyncOptions::kSyncPoint
,
1824 argGroup().immPtr(inst
->extra
<LdFuncCached
>()->name
)
1827 m_as
. bind (&noLookup
);
1830 void CodeGenerator::cgLdStackAddr(IRInstruction
* inst
) {
1831 auto const dstReg
= x2a(dstLoc(0).reg());
1832 auto const baseReg
= x2a(srcLoc(0).reg());
1833 auto const offset
= cellsToBytes(inst
->extra
<LdStackAddr
>()->offset
);
1834 emitRegGetsRegPlusImm(m_as
, dstReg
, baseReg
, offset
);
1837 void CodeGenerator::cgSpillStack(IRInstruction
* inst
) {
1838 // TODO(2966414): so much of this logic could be shared. The opcode itself
1839 // should probably be broken up.
1840 auto const spDeficit
= inst
->src(1)->intVal();
1841 auto const spillVals
= inst
->srcs().subpiece(2);
1842 auto const numSpillSrcs
= spillVals
.size();
1843 auto const dstReg
= x2a(dstLoc(0).reg());
1844 auto const spReg
= x2a(srcLoc(0).reg());
1845 auto const spillCells
= spillValueCells(inst
);
1847 int64_t adjustment
= (spDeficit
- spillCells
) * sizeof(Cell
);
1848 for (uint32_t i
= 0; i
< numSpillSrcs
; ++i
) {
1849 const int64_t offset
= i
* sizeof(Cell
) + adjustment
;
1850 emitStore(spReg
, offset
, spillVals
[i
], srcLoc(i
+ 2));
1852 emitRegGetsRegPlusImm(m_as
, dstReg
, spReg
, adjustment
);
1855 void CodeGenerator::cgInterpOneCommon(IRInstruction
* inst
) {
1856 auto pcOff
= inst
->extra
<InterpOneData
>()->bcOff
;
1858 auto opc
= *(curFunc()->unit()->at(pcOff
));
1859 auto* interpOneHelper
= interpOneEntryPoints
[opc
];
1862 CppCall(interpOneHelper
),
1863 callDest(InvalidReg
),
1864 SyncOptions::kSyncPoint
,
1865 argGroup().ssa(1/*fp*/).ssa(0/*sp*/).imm(pcOff
));
1868 void CodeGenerator::cgInterpOne(IRInstruction
* inst
) {
1869 cgInterpOneCommon(inst
);
1871 auto const& extra
= *inst
->extra
<InterpOne
>();
1872 auto newSpReg
= x2a(dstLoc(0).reg());
1874 auto spAdjustBytes
= cellsToBytes(extra
.cellsPopped
- extra
.cellsPushed
);
1875 emitRegGetsRegPlusImm(m_as
, newSpReg
, newSpReg
, spAdjustBytes
);
1878 void CodeGenerator::cgInterpOneCF(IRInstruction
* inst
) {
1879 cgInterpOneCommon(inst
);
1881 m_as
. Ldr (rVmFp
, rReturnReg
[offsetof(ExecutionContext
, m_fp
)]);
1882 m_as
. Ldr (rVmSp
, rReturnReg
[offsetof(ExecutionContext
, m_stack
) +
1883 Stack::topOfStackOffset()]);
1885 emitServiceReq(mcg
->code
.main(), REQ_RESUME
);
1888 //////////////////////////////////////////////////////////////////////
1890 Address
CodeGenerator::cgInst(IRInstruction
* inst
) {
1891 Opcode opc
= inst
->op();
1892 auto const start
= m_as
.frontier();
1894 SCOPE_EXIT
{ m_curInst
= nullptr; };
1897 #define O(name, dsts, srcs, flags) \
1898 case name: FTRACE(7, "cg" #name "\n"); \
1899 cg ## name (inst); \
1900 return m_as.frontier() == start ? nullptr : start;
