Bug 1834537 - Part 1: Simplify JIT nursery allocation r=jandem
[gecko.git] / js / src / jit / IonIC.cpp
blobb6536d6d83489d1d6816b250e6dfae84f36f1f39
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "jit/IonIC.h"
9 #include "jit/CacheIRCompiler.h"
10 #include "jit/CacheIRGenerator.h"
11 #include "jit/IonScript.h"
12 #include "jit/VMFunctions.h"
13 #include "util/DiagnosticAssertions.h"
14 #include "vm/EqualityOperations.h"
15 #include "vm/Iteration.h"
17 #include "vm/Interpreter-inl.h"
18 #include "vm/JSScript-inl.h"
20 using namespace js;
21 using namespace js::jit;
23 void IonIC::resetCodeRaw(IonScript* ionScript) {
24 codeRaw_ = fallbackAddr(ionScript);
27 uint8_t* IonIC::fallbackAddr(IonScript* ionScript) const {
28 return ionScript->method()->raw() + fallbackOffset_;
31 uint8_t* IonIC::rejoinAddr(IonScript* ionScript) const {
32 return ionScript->method()->raw() + rejoinOffset_;
35 Register IonIC::scratchRegisterForEntryJump() {
36 switch (kind_) {
37 case CacheKind::GetProp:
38 case CacheKind::GetElem:
39 return asGetPropertyIC()->output().scratchReg();
40 case CacheKind::GetPropSuper:
41 case CacheKind::GetElemSuper:
42 return asGetPropSuperIC()->output().scratchReg();
43 case CacheKind::SetProp:
44 case CacheKind::SetElem:
45 return asSetPropertyIC()->temp();
46 case CacheKind::GetName:
47 return asGetNameIC()->temp();
48 case CacheKind::BindName:
49 return asBindNameIC()->temp();
50 case CacheKind::In:
51 return asInIC()->temp();
52 case CacheKind::HasOwn:
53 return asHasOwnIC()->output();
54 case CacheKind::CheckPrivateField:
55 return asCheckPrivateFieldIC()->output();
56 case CacheKind::GetIterator:
57 return asGetIteratorIC()->temp1();
58 case CacheKind::OptimizeSpreadCall:
59 return asOptimizeSpreadCallIC()->temp();
60 case CacheKind::InstanceOf:
61 return asInstanceOfIC()->output();
62 case CacheKind::UnaryArith:
63 return asUnaryArithIC()->output().scratchReg();
64 case CacheKind::ToPropertyKey:
65 return asToPropertyKeyIC()->output().scratchReg();
66 case CacheKind::BinaryArith:
67 return asBinaryArithIC()->output().scratchReg();
68 case CacheKind::Compare:
69 return asCompareIC()->output();
70 case CacheKind::CloseIter:
71 return asCloseIterIC()->temp();
72 case CacheKind::Call:
73 case CacheKind::TypeOf:
74 case CacheKind::ToBool:
75 case CacheKind::GetIntrinsic:
76 case CacheKind::NewArray:
77 case CacheKind::NewObject:
78 MOZ_CRASH("Unsupported IC");
81 MOZ_CRASH("Invalid kind");
84 void IonIC::discardStubs(Zone* zone, IonScript* ionScript) {
85 if (firstStub_) {
86 // We are removing edges from IonIC to gcthings. Perform a write barrier to
87 // let the GC know about those edges.
88 PreWriteBarrier(zone, ionScript);
91 #ifdef JS_CRASH_DIAGNOSTICS
92 IonICStub* stub = firstStub_;
93 while (stub) {
94 IonICStub* next = stub->next();
95 stub->poison();
96 stub = next;
98 #endif
100 firstStub_ = nullptr;
101 resetCodeRaw(ionScript);
102 state_.trackUnlinkedAllStubs();
105 void IonIC::reset(Zone* zone, IonScript* ionScript) {
106 discardStubs(zone, ionScript);
107 state_.reset();
110 void IonIC::trace(JSTracer* trc, IonScript* ionScript) {
111 if (script_) {
112 TraceManuallyBarrieredEdge(trc, &script_, "IonIC::script_");
115 uint8_t* nextCodeRaw = codeRaw_;
116 for (IonICStub* stub = firstStub_; stub; stub = stub->next()) {
117 JitCode* code = JitCode::FromExecutable(nextCodeRaw);
118 TraceManuallyBarrieredEdge(trc, &code, "ion-ic-code");
120 TraceCacheIRStub(trc, stub, stub->stubInfo());
122 nextCodeRaw = stub->nextCodeRaw();
125 MOZ_ASSERT(nextCodeRaw == fallbackAddr(ionScript));
128 // This helper handles ICState updates/transitions while attaching CacheIR
129 // stubs.
130 template <typename IRGenerator, typename... Args>
131 static void TryAttachIonStub(JSContext* cx, IonIC* ic, IonScript* ionScript,
132 Args&&... args) {
133 if (ic->state().maybeTransition()) {
134 ic->discardStubs(cx->zone(), ionScript);
137 if (ic->state().canAttachStub()) {
138 RootedScript script(cx, ic->script());
139 bool attached = false;
140 IRGenerator gen(cx, script, ic->pc(), ic->state(),
141 std::forward<Args>(args)...);
142 switch (gen.tryAttachStub()) {
143 case AttachDecision::Attach:
144 ic->attachCacheIRStub(cx, gen.writerRef(), gen.cacheKind(), ionScript,
145 &attached);
146 break;
147 case AttachDecision::NoAction:
148 break;
149 case AttachDecision::TemporarilyUnoptimizable:
150 attached = true;
151 break;
152 case AttachDecision::Deferred:
153 MOZ_ASSERT_UNREACHABLE("Not expected in generic TryAttachIonStub");
154 break;
156 if (!attached) {
157 ic->state().trackNotAttached();
162 /* static */
163 bool IonGetPropertyIC::update(JSContext* cx, HandleScript outerScript,
164 IonGetPropertyIC* ic, HandleValue val,
165 HandleValue idVal, MutableHandleValue res) {
166 IonScript* ionScript = outerScript->ionScript();
168 // Optimized-arguments and other magic values must not escape to Ion ICs.
169 MOZ_ASSERT(!val.isMagic());
171 TryAttachIonStub<GetPropIRGenerator>(cx, ic, ionScript, ic->kind(), val,
172 idVal);
174 if (ic->kind() == CacheKind::GetProp) {
175 Rooted<PropertyName*> name(cx, idVal.toString()->asAtom().asPropertyName());
176 if (!GetProperty(cx, val, name, res)) {
177 return false;
179 } else {
180 MOZ_ASSERT(ic->kind() == CacheKind::GetElem);
181 if (!GetElementOperation(cx, val, idVal, res)) {
182 return false;
186 return true;
189 /* static */
190 bool IonGetPropSuperIC::update(JSContext* cx, HandleScript outerScript,
191 IonGetPropSuperIC* ic, HandleObject obj,
192 HandleValue receiver, HandleValue idVal,
193 MutableHandleValue res) {
194 IonScript* ionScript = outerScript->ionScript();
196 if (ic->state().maybeTransition()) {
197 ic->discardStubs(cx->zone(), ionScript);
200 RootedValue val(cx, ObjectValue(*obj));
202 TryAttachIonStub<GetPropIRGenerator>(cx, ic, ionScript, ic->kind(), val,
203 idVal);
205 if (ic->kind() == CacheKind::GetPropSuper) {
206 Rooted<PropertyName*> name(cx, idVal.toString()->asAtom().asPropertyName());
207 if (!GetProperty(cx, obj, receiver, name, res)) {
208 return false;
210 } else {
211 MOZ_ASSERT(ic->kind() == CacheKind::GetElemSuper);
213 JSOp op = JSOp(*ic->pc());
214 MOZ_ASSERT(op == JSOp::GetElemSuper);
216 if (!GetObjectElementOperation(cx, op, obj, receiver, idVal, res)) {
217 return false;
221 return true;
224 /* static */
225 bool IonSetPropertyIC::update(JSContext* cx, HandleScript outerScript,
226 IonSetPropertyIC* ic, HandleObject obj,
227 HandleValue idVal, HandleValue rhs) {
228 using DeferType = SetPropIRGenerator::DeferType;
230 Rooted<Shape*> oldShape(cx);
231 IonScript* ionScript = outerScript->ionScript();
233 bool attached = false;
234 DeferType deferType = DeferType::None;
236 if (ic->state().maybeTransition()) {
237 ic->discardStubs(cx->zone(), ionScript);
240 if (ic->state().canAttachStub()) {
241 oldShape = obj->shape();
243 RootedValue objv(cx, ObjectValue(*obj));
244 RootedScript script(cx, ic->script());
245 jsbytecode* pc = ic->pc();
247 SetPropIRGenerator gen(cx, script, pc, ic->kind(), ic->state(), objv, idVal,
248 rhs);
249 switch (gen.tryAttachStub()) {
250 case AttachDecision::Attach:
251 ic->attachCacheIRStub(cx, gen.writerRef(), gen.cacheKind(), ionScript,
252 &attached);
253 break;
254 case AttachDecision::NoAction:
255 break;
256 case AttachDecision::TemporarilyUnoptimizable:
257 attached = true;
258 break;
259 case AttachDecision::Deferred:
260 deferType = gen.deferType();
261 MOZ_ASSERT(deferType != DeferType::None);
262 break;
266 jsbytecode* pc = ic->pc();
267 if (ic->kind() == CacheKind::SetElem) {
268 if (JSOp(*pc) == JSOp::InitElemInc) {
269 if (!InitElemIncOperation(cx, obj.as<ArrayObject>(), idVal.toInt32(),
270 rhs)) {
271 return false;
273 } else if (IsPropertyInitOp(JSOp(*pc))) {
274 if (!InitElemOperation(cx, pc, obj, idVal, rhs)) {
275 return false;
277 } else {
278 MOZ_ASSERT(IsPropertySetOp(JSOp(*pc)));
279 if (!SetObjectElement(cx, obj, idVal, rhs, ic->strict())) {
280 return false;
283 } else {
284 MOZ_ASSERT(ic->kind() == CacheKind::SetProp);
286 if (JSOp(*pc) == JSOp::InitGLexical) {
287 RootedScript script(cx, ic->script());
288 MOZ_ASSERT(!script->hasNonSyntacticScope());
289 InitGlobalLexicalOperation(cx, &cx->global()->lexicalEnvironment(),
290 script, pc, rhs);
291 } else if (IsPropertyInitOp(JSOp(*pc))) {
292 Rooted<PropertyName*> name(cx,
293 idVal.toString()->asAtom().asPropertyName());
294 if (!InitPropertyOperation(cx, pc, obj, name, rhs)) {
295 return false;
297 } else {
298 MOZ_ASSERT(IsPropertySetOp(JSOp(*pc)));
299 Rooted<PropertyName*> name(cx,
300 idVal.toString()->asAtom().asPropertyName());
301 if (!SetProperty(cx, obj, name, rhs, ic->strict(), pc)) {
302 return false;
307 if (attached) {
308 return true;
311 // The SetProperty call might have entered this IC recursively, so try
312 // to transition.
313 if (ic->state().maybeTransition()) {
314 ic->discardStubs(cx->zone(), ionScript);
317 bool canAttachStub = ic->state().canAttachStub();
318 if (deferType != DeferType::None && canAttachStub) {
319 RootedValue objv(cx, ObjectValue(*obj));
320 RootedScript script(cx, ic->script());
321 jsbytecode* pc = ic->pc();
322 SetPropIRGenerator gen(cx, script, pc, ic->kind(), ic->state(), objv, idVal,
323 rhs);
324 MOZ_ASSERT(deferType == DeferType::AddSlot);
325 AttachDecision decision = gen.tryAttachAddSlotStub(oldShape);
327 switch (decision) {
328 case AttachDecision::Attach:
329 ic->attachCacheIRStub(cx, gen.writerRef(), gen.cacheKind(), ionScript,
330 &attached);
331 break;
332 case AttachDecision::NoAction:
333 gen.trackAttached(IRGenerator::NotAttached);
334 break;
335 case AttachDecision::TemporarilyUnoptimizable:
336 case AttachDecision::Deferred:
337 MOZ_ASSERT_UNREACHABLE("Invalid attach result");
338 break;
341 if (!attached && canAttachStub) {
342 ic->state().trackNotAttached();
345 return true;
348 /* static */
349 bool IonGetNameIC::update(JSContext* cx, HandleScript outerScript,
350 IonGetNameIC* ic, HandleObject envChain,
351 MutableHandleValue res) {
352 IonScript* ionScript = outerScript->ionScript();
353 jsbytecode* pc = ic->pc();
354 Rooted<PropertyName*> name(cx, ic->script()->getName(pc));
356 TryAttachIonStub<GetNameIRGenerator>(cx, ic, ionScript, envChain, name);
358 RootedObject obj(cx);
359 RootedObject holder(cx);
360 PropertyResult prop;
361 if (!LookupName(cx, name, envChain, &obj, &holder, &prop)) {
362 return false;
365 if (JSOp(*GetNextPc(pc)) == JSOp::Typeof) {
366 return FetchName<GetNameMode::TypeOf>(cx, obj, holder, name, prop, res);
369 return FetchName<GetNameMode::Normal>(cx, obj, holder, name, prop, res);
372 /* static */
373 JSObject* IonBindNameIC::update(JSContext* cx, HandleScript outerScript,
374 IonBindNameIC* ic, HandleObject envChain) {
375 IonScript* ionScript = outerScript->ionScript();
376 jsbytecode* pc = ic->pc();
377 Rooted<PropertyName*> name(cx, ic->script()->getName(pc));
379 TryAttachIonStub<BindNameIRGenerator>(cx, ic, ionScript, envChain, name);
381 RootedObject holder(cx);
382 if (!LookupNameUnqualified(cx, name, envChain, &holder)) {
383 return nullptr;
386 return holder;
389 /* static */
390 JSObject* IonGetIteratorIC::update(JSContext* cx, HandleScript outerScript,
391 IonGetIteratorIC* ic, HandleValue value) {
392 IonScript* ionScript = outerScript->ionScript();
394 TryAttachIonStub<GetIteratorIRGenerator>(cx, ic, ionScript, value);
396 PropertyIteratorObject* iterObj = ValueToIterator(cx, value);
397 if (!iterObj) {
398 return nullptr;
401 return iterObj;
404 /* static */
405 bool IonOptimizeSpreadCallIC::update(JSContext* cx, HandleScript outerScript,
406 IonOptimizeSpreadCallIC* ic,
407 HandleValue value,
408 MutableHandleValue result) {
409 IonScript* ionScript = outerScript->ionScript();
411 TryAttachIonStub<OptimizeSpreadCallIRGenerator>(cx, ic, ionScript, value);
413 return OptimizeSpreadCall(cx, value, result);
416 /* static */
417 bool IonHasOwnIC::update(JSContext* cx, HandleScript outerScript,
418 IonHasOwnIC* ic, HandleValue val, HandleValue idVal,
419 int32_t* res) {
420 IonScript* ionScript = outerScript->ionScript();
422 TryAttachIonStub<HasPropIRGenerator>(cx, ic, ionScript, CacheKind::HasOwn,
423 idVal, val);
425 bool found;
426 if (!HasOwnProperty(cx, val, idVal, &found)) {
427 return false;
430 *res = found;
431 return true;
434 /* static */
435 bool IonCheckPrivateFieldIC::update(JSContext* cx, HandleScript outerScript,
436 IonCheckPrivateFieldIC* ic, HandleValue val,
437 HandleValue idVal, bool* res) {
438 IonScript* ionScript = outerScript->ionScript();
439 jsbytecode* pc = ic->pc();
441 TryAttachIonStub<CheckPrivateFieldIRGenerator>(
442 cx, ic, ionScript, CacheKind::CheckPrivateField, idVal, val);
444 return CheckPrivateFieldOperation(cx, pc, val, idVal, res);
447 /* static */
448 bool IonInIC::update(JSContext* cx, HandleScript outerScript, IonInIC* ic,
449 HandleValue key, HandleObject obj, bool* res) {
450 IonScript* ionScript = outerScript->ionScript();
451 RootedValue objV(cx, ObjectValue(*obj));
453 TryAttachIonStub<HasPropIRGenerator>(cx, ic, ionScript, CacheKind::In, key,
454 objV);
456 return OperatorIn(cx, key, obj, res);
458 /* static */
459 bool IonInstanceOfIC::update(JSContext* cx, HandleScript outerScript,
460 IonInstanceOfIC* ic, HandleValue lhs,
461 HandleObject rhs, bool* res) {
462 IonScript* ionScript = outerScript->ionScript();
464 TryAttachIonStub<InstanceOfIRGenerator>(cx, ic, ionScript, lhs, rhs);
466 return InstanceofOperator(cx, rhs, lhs, res);
469 /* static */
470 bool IonToPropertyKeyIC::update(JSContext* cx, HandleScript outerScript,
471 IonToPropertyKeyIC* ic, HandleValue val,
472 MutableHandleValue res) {
473 IonScript* ionScript = outerScript->ionScript();
475 TryAttachIonStub<ToPropertyKeyIRGenerator>(cx, ic, ionScript, val);
477 return ToPropertyKeyOperation(cx, val, res);
480 /* static */
481 bool IonCloseIterIC::update(JSContext* cx, HandleScript outerScript,
482 IonCloseIterIC* ic, HandleObject iter) {
483 IonScript* ionScript = outerScript->ionScript();
484 CompletionKind kind = ic->completionKind();
486 TryAttachIonStub<CloseIterIRGenerator>(cx, ic, ionScript, iter, kind);
488 return CloseIterOperation(cx, iter, kind);
491 /* static */
492 bool IonUnaryArithIC::update(JSContext* cx, HandleScript outerScript,
493 IonUnaryArithIC* ic, HandleValue val,
494 MutableHandleValue res) {
495 IonScript* ionScript = outerScript->ionScript();
496 RootedScript script(cx, ic->script());
497 jsbytecode* pc = ic->pc();
498 JSOp op = JSOp(*pc);
500 switch (op) {
501 case JSOp::BitNot: {
502 res.set(val);
503 if (!BitNot(cx, res, res)) {
504 return false;
506 break;
508 case JSOp::Pos: {
509 res.set(val);
510 if (!ToNumber(cx, res)) {
511 return false;
513 break;
515 case JSOp::Neg: {
516 res.set(val);
517 if (!NegOperation(cx, res, res)) {
518 return false;
520 break;
522 case JSOp::Inc: {
523 if (!IncOperation(cx, val, res)) {
524 return false;
526 break;
528 case JSOp::Dec: {
529 if (!DecOperation(cx, val, res)) {
530 return false;
532 break;
534 case JSOp::ToNumeric: {
535 res.set(val);
536 if (!ToNumeric(cx, res)) {
537 return false;
539 break;
541 default:
542 MOZ_CRASH("Unexpected op");
544 MOZ_ASSERT(res.isNumeric());
546 TryAttachIonStub<UnaryArithIRGenerator>(cx, ic, ionScript, op, val, res);
548 return true;
551 /* static */
552 bool IonBinaryArithIC::update(JSContext* cx, HandleScript outerScript,
553 IonBinaryArithIC* ic, HandleValue lhs,
554 HandleValue rhs, MutableHandleValue ret) {
555 IonScript* ionScript = outerScript->ionScript();
556 RootedScript script(cx, ic->script());
557 jsbytecode* pc = ic->pc();
558 JSOp op = JSOp(*pc);
560 // Don't pass lhs/rhs directly, we need the original values when
561 // generating stubs.
562 RootedValue lhsCopy(cx, lhs);
563 RootedValue rhsCopy(cx, rhs);
565 // Perform the compare operation.
566 switch (op) {
567 case JSOp::Add:
568 // Do an add.
569 if (!AddValues(cx, &lhsCopy, &rhsCopy, ret)) {
570 return false;
572 break;
573 case JSOp::Sub:
574 if (!SubValues(cx, &lhsCopy, &rhsCopy, ret)) {
575 return false;
577 break;
578 case JSOp::Mul:
579 if (!MulValues(cx, &lhsCopy, &rhsCopy, ret)) {
580 return false;
582 break;
583 case JSOp::Div:
584 if (!DivValues(cx, &lhsCopy, &rhsCopy, ret)) {
585 return false;
587 break;
588 case JSOp::Mod:
589 if (!ModValues(cx, &lhsCopy, &rhsCopy, ret)) {
590 return false;
592 break;
593 case JSOp::Pow:
594 if (!PowValues(cx, &lhsCopy, &rhsCopy, ret)) {
595 return false;
597 break;
598 case JSOp::BitOr: {
599 if (!BitOr(cx, &lhsCopy, &rhsCopy, ret)) {
600 return false;
602 break;
604 case JSOp::BitXor: {
605 if (!BitXor(cx, &lhsCopy, &rhsCopy, ret)) {
606 return false;
608 break;
610 case JSOp::BitAnd: {
611 if (!BitAnd(cx, &lhsCopy, &rhsCopy, ret)) {
612 return false;
614 break;
616 case JSOp::Lsh: {
617 if (!BitLsh(cx, &lhsCopy, &rhsCopy, ret)) {
618 return false;
620 break;
622 case JSOp::Rsh: {
623 if (!BitRsh(cx, &lhsCopy, &rhsCopy, ret)) {
624 return false;
626 break;
628 case JSOp::Ursh: {
629 if (!UrshValues(cx, &lhsCopy, &rhsCopy, ret)) {
630 return false;
632 break;
634 default:
635 MOZ_CRASH("Unhandled binary arith op");
638 TryAttachIonStub<BinaryArithIRGenerator>(cx, ic, ionScript, op, lhs, rhs,
639 ret);
641 return true;
644 /* static */
645 bool IonCompareIC::update(JSContext* cx, HandleScript outerScript,
646 IonCompareIC* ic, HandleValue lhs, HandleValue rhs,
647 bool* res) {
648 IonScript* ionScript = outerScript->ionScript();
649 RootedScript script(cx, ic->script());
650 jsbytecode* pc = ic->pc();
651 JSOp op = JSOp(*pc);
653 // Don't pass lhs/rhs directly, we need the original values when
654 // generating stubs.
655 RootedValue lhsCopy(cx, lhs);
656 RootedValue rhsCopy(cx, rhs);
658 // Perform the compare operation.
659 switch (op) {
660 case JSOp::Lt:
661 if (!LessThan(cx, &lhsCopy, &rhsCopy, res)) {
662 return false;
664 break;
665 case JSOp::Le:
666 if (!LessThanOrEqual(cx, &lhsCopy, &rhsCopy, res)) {
667 return false;
669 break;
670 case JSOp::Gt:
671 if (!GreaterThan(cx, &lhsCopy, &rhsCopy, res)) {
672 return false;
674 break;
675 case JSOp::Ge:
676 if (!GreaterThanOrEqual(cx, &lhsCopy, &rhsCopy, res)) {
677 return false;
679 break;
680 case JSOp::Eq:
681 if (!js::LooselyEqual(cx, lhsCopy, rhsCopy, res)) {
682 return false;
684 break;
685 case JSOp::Ne:
686 if (!js::LooselyEqual(cx, lhsCopy, rhsCopy, res)) {
687 return false;
689 *res = !*res;
690 break;
691 case JSOp::StrictEq:
692 if (!js::StrictlyEqual(cx, lhsCopy, rhsCopy, res)) {
693 return false;
695 break;
696 case JSOp::StrictNe:
697 if (!js::StrictlyEqual(cx, lhsCopy, rhsCopy, res)) {
698 return false;
700 *res = !*res;
701 break;
702 default:
703 MOZ_ASSERT_UNREACHABLE("Unhandled ion compare op");
704 return false;
707 TryAttachIonStub<CompareIRGenerator>(cx, ic, ionScript, op, lhs, rhs);
709 return true;
712 uint8_t* IonICStub::stubDataStart() {
713 return reinterpret_cast<uint8_t*>(this) + stubInfo_->stubDataOffset();
716 void IonIC::attachStub(IonICStub* newStub, JitCode* code) {
717 MOZ_ASSERT(newStub);
718 MOZ_ASSERT(code);
720 if (firstStub_) {
721 newStub->setNext(firstStub_, codeRaw_);
723 firstStub_ = newStub;
724 codeRaw_ = code->raw();
726 state_.trackAttached();