3 ** Copyright (C) 2005-2010 Mike Pall. See Copyright Notice in luajit.h
21 #include "lj_target.h"
23 /* Some local macros to save typing. Undef'd at the end. */
24 #define IR(ref) (&J->cur.ir[(ref)])
26 /* -- Snapshot buffer allocation ------------------------------------------ */
28 /* Grow snapshot buffer. */
29 void lj_snap_grow_buf_(jit_State
*J
, MSize need
)
31 MSize maxsnap
= (MSize
)J
->param
[JIT_P_maxsnap
];
33 lj_trace_err(J
, LJ_TRERR_SNAPOV
);
34 lj_mem_growvec(J
->L
, J
->snapbuf
, J
->sizesnap
, maxsnap
, SnapShot
);
35 J
->cur
.snap
= J
->snapbuf
;
38 /* Grow snapshot map buffer. */
39 void lj_snap_grow_map_(jit_State
*J
, MSize need
)
41 if (need
< 2*J
->sizesnapmap
)
42 need
= 2*J
->sizesnapmap
;
45 J
->snapmapbuf
= (SnapEntry
*)lj_mem_realloc(J
->L
, J
->snapmapbuf
,
46 J
->sizesnapmap
*sizeof(SnapEntry
), need
*sizeof(SnapEntry
));
47 J
->cur
.snapmap
= J
->snapmapbuf
;
48 J
->sizesnapmap
= need
;
51 /* -- Snapshot generation ------------------------------------------------- */
53 /* Add all modified slots to the snapshot. */
54 static MSize
snapshot_slots(jit_State
*J
, SnapEntry
*map
, BCReg nslots
)
56 IRRef retf
= J
->chain
[IR_RETF
]; /* Limits SLOAD restore elimination. */
59 for (s
= 0; s
< nslots
; s
++) {
61 IRRef ref
= tref_ref(tr
);
63 SnapEntry sn
= SNAP_TR(s
, tr
);
65 if (ir
->o
== IR_SLOAD
&& ir
->op1
== s
&& ref
> retf
) {
66 /* No need to snapshot unmodified non-inherited slots. */
67 if (!(ir
->op2
& IRSLOAD_INHERIT
))
69 /* No need to restore readonly slots and unmodified non-parent slots. */
70 if ((ir
->op2
& (IRSLOAD_READONLY
|IRSLOAD_PARENT
)) != IRSLOAD_PARENT
)
79 /* Add frame links at the end of the snapshot. */
80 static void snapshot_framelinks(jit_State
*J
, SnapEntry
*map
)
82 cTValue
*frame
= J
->L
->base
- 1;
83 cTValue
*lim
= J
->L
->base
- J
->baseslot
;
85 map
[f
++] = SNAP_MKPC(J
->pc
); /* The current PC is always the first entry. */
86 while (frame
> lim
) { /* Backwards traversal of all frames above base. */
87 if (frame_islua(frame
)) {
88 map
[f
++] = SNAP_MKPC(frame_pc(frame
));
89 frame
= frame_prevl(frame
);
90 } else if (frame_ispcall(frame
)) {
91 map
[f
++] = SNAP_MKFTSZ(frame_ftsz(frame
));
92 frame
= frame_prevd(frame
);
93 } else if (frame_iscont(frame
)) {
94 map
[f
++] = SNAP_MKFTSZ(frame_ftsz(frame
));
95 map
[f
++] = SNAP_MKPC(frame_contpc(frame
));
96 frame
= frame_prevd(frame
);
101 lua_assert(f
== (MSize
)(1 + J
->framedepth
));
104 /* Take a snapshot of the current stack. */
105 static void snapshot_stack(jit_State
*J
, SnapShot
*snap
, MSize nsnapmap
)
107 BCReg nslots
= J
->baseslot
+ J
->maxslot
;
110 /* Conservative estimate. */
111 lj_snap_grow_map(J
, nsnapmap
+ nslots
+ (MSize
)J
->framedepth
+1);
112 p
= &J
->cur
.snapmap
[nsnapmap
];
113 nent
= snapshot_slots(J
, p
, nslots
);
114 snapshot_framelinks(J
, p
+ nent
);
115 snap
->mapofs
= (uint16_t)nsnapmap
;
116 snap
->ref
= (IRRef1
)J
->cur
.nins
;
117 snap
->nent
= (uint8_t)nent
;
118 snap
->depth
= (uint8_t)J
->framedepth
;
119 snap
->nslots
= (uint8_t)nslots
;
121 J
->cur
.nsnapmap
= (uint16_t)(nsnapmap
+ nent
+ 1 + J
->framedepth
);
124 /* Add or merge a snapshot. */
125 void lj_snap_add(jit_State
*J
)
127 MSize nsnap
= J
->cur
.nsnap
;
128 MSize nsnapmap
= J
->cur
.nsnapmap
;
129 /* Merge if no ins. inbetween or if requested and no guard inbetween. */
130 if (J
->mergesnap
? !irt_isguard(J
->guardemit
) :
131 (nsnap
> 0 && J
->cur
.snap
[nsnap
-1].ref
== J
->cur
.nins
)) {
132 nsnapmap
= J
->cur
.snap
[--nsnap
].mapofs
;
134 lj_snap_grow_buf(J
, nsnap
+1);
135 J
->cur
.nsnap
= (uint16_t)(nsnap
+1);
138 J
->guardemit
.irt
= 0;
139 snapshot_stack(J
, &J
->cur
.snap
[nsnap
], nsnapmap
);
142 /* Shrink last snapshot. */
143 void lj_snap_shrink(jit_State
*J
)
145 BCReg nslots
= J
->baseslot
+ J
->maxslot
;
146 SnapShot
*snap
= &J
->cur
.snap
[J
->cur
.nsnap
-1];
147 SnapEntry
*map
= &J
->cur
.snapmap
[snap
->mapofs
];
148 MSize nent
= snap
->nent
;
149 lua_assert(nslots
< snap
->nslots
);
150 snap
->nslots
= (uint8_t)nslots
;
151 if (nent
> 0 && snap_slot(map
[nent
-1]) >= nslots
) {
152 MSize s
, delta
, depth
= snap
->depth
;
153 lua_assert(depth
== (MSize
)J
->framedepth
);
154 for (nent
--; nent
> 0 && snap_slot(map
[nent
-1]) >= nslots
; nent
--)
156 delta
= snap
->nent
- nent
;
157 snap
->nent
= (uint8_t)nent
;
158 J
->cur
.nsnapmap
= (uint16_t)(snap
->mapofs
+ nent
+ 1 + depth
);
160 for (s
= 0; s
<= depth
; s
++) /* Move PC + frame links down. */
161 map
[s
] = map
[s
+delta
];
165 /* -- Snapshot access ----------------------------------------------------- */
167 /* Initialize a Bloom Filter with all renamed refs.
168 ** There are very few renames (often none), so the filter has
169 ** very few bits set. This makes it suitable for negative filtering.
171 static BloomFilter
snap_renamefilter(Trace
*T
, SnapNo lim
)
173 BloomFilter rfilt
= 0;
175 for (ir
= &T
->ir
[T
->nins
-1]; ir
->o
== IR_RENAME
; ir
--)
177 bloomset(rfilt
, ir
->op1
);
181 /* Process matching renames to find the original RegSP. */
182 static RegSP
snap_renameref(Trace
*T
, SnapNo lim
, IRRef ref
, RegSP rs
)
185 for (ir
= &T
->ir
[T
->nins
-1]; ir
->o
== IR_RENAME
; ir
--)
186 if (ir
->op1
== ref
&& ir
->op2
<= lim
)
191 /* Convert a snapshot into a linear slot -> RegSP map.
192 ** Note: unused slots are not initialized!
194 void lj_snap_regspmap(uint16_t *rsmap
, Trace
*T
, SnapNo snapno
)
196 SnapShot
*snap
= &T
->snap
[snapno
];
197 MSize n
, nent
= snap
->nent
;
198 SnapEntry
*map
= &T
->snapmap
[snap
->mapofs
];
199 BloomFilter rfilt
= snap_renamefilter(T
, snapno
);
200 for (n
= 0; n
< nent
; n
++) {
201 SnapEntry sn
= map
[n
];
202 IRRef ref
= snap_ref(sn
);
203 if (!irref_isk(ref
)) {
204 IRIns
*ir
= &T
->ir
[ref
];
205 uint32_t rs
= ir
->prev
;
206 if (bloomtest(rfilt
, ref
))
207 rs
= snap_renameref(T
, snapno
, ref
, rs
);
208 rsmap
[snap_slot(sn
)] = (uint16_t)rs
;
213 /* Restore interpreter state from exit state with the help of a snapshot. */
214 const BCIns
*lj_snap_restore(jit_State
*J
, void *exptr
)
216 ExitState
*ex
= (ExitState
*)exptr
;
217 SnapNo snapno
= J
->exitno
; /* For now, snapno == exitno. */
218 Trace
*T
= J
->trace
[J
->parent
];
219 SnapShot
*snap
= &T
->snap
[snapno
];
220 MSize n
, nent
= snap
->nent
;
221 SnapEntry
*map
= &T
->snapmap
[snap
->mapofs
];
222 SnapEntry
*flinks
= map
+ nent
+ snap
->depth
;
224 BCReg nslots
= snap
->nslots
;
226 BloomFilter rfilt
= snap_renamefilter(T
, snapno
);
227 const BCIns
*pc
= snap_pc(map
[nent
]);
230 /* Set interpreter PC to the next PC to get correct error messages. */
231 setcframe_pc(cframe_raw(L
->cframe
), pc
+1);
233 /* Make sure the stack is big enough for the slots from the snapshot. */
234 if (LJ_UNLIKELY(L
->base
+ nslots
> L
->maxstack
)) {
235 L
->top
= curr_topL(L
);
236 lj_state_growstack(L
, nslots
- curr_proto(L
)->framesize
);
239 /* Fill stack slots with data from the registers and spill slots. */
241 ftsz0
= frame_ftsz(frame
); /* Preserve link to previous frame in slot #0. */
242 for (n
= 0; n
< nent
; n
++) {
243 SnapEntry sn
= map
[n
];
244 IRRef ref
= snap_ref(sn
);
245 BCReg s
= snap_slot(sn
);
246 TValue
*o
= &frame
[s
]; /* Stack slots are relative to start frame. */
247 IRIns
*ir
= &T
->ir
[ref
];
248 if (irref_isk(ref
)) { /* Restore constant slot. */
249 lj_ir_kvalue(L
, o
, ir
);
250 if ((sn
& (SNAP_CONT
|SNAP_FRAME
))) {
251 /* Overwrite tag with frame link. */
252 o
->fr
.tp
.ftsz
= s
!= 0 ? (int32_t)*flinks
-- : ftsz0
;
253 if ((sn
& SNAP_FRAME
)) {
254 GCfunc
*fn
= ir_kfunc(ir
);
256 MSize framesize
= funcproto(fn
)->framesize
;
259 if (LJ_UNLIKELY(o
+ framesize
> L
->maxstack
)) { /* Grow again? */
260 ptrdiff_t fsave
= savestack(L
, frame
);
262 lj_state_growstack(L
, framesize
);
263 frame
= restorestack(L
, fsave
);
270 } else if (!(sn
& SNAP_NORESTORE
)) {
273 lua_assert(!(sn
& (SNAP_CONT
|SNAP_FRAME
)));
274 if (LJ_UNLIKELY(bloomtest(rfilt
, ref
)))
275 rs
= snap_renameref(T
, snapno
, ref
, rs
);
276 if (ra_hasspill(regsp_spill(rs
))) { /* Restore from spill slot. */
277 int32_t *sps
= &ex
->spill
[regsp_spill(rs
)];
278 if (irt_isinteger(t
)) {
280 } else if (irt_isnum(t
)) {
281 o
->u64
= *(uint64_t *)sps
;
283 } else if (irt_islightud(t
)) {
284 /* 64 bit lightuserdata which may escape already has the tag bits. */
285 o
->u64
= *(uint64_t *)sps
;
288 lua_assert(!irt_ispri(t
)); /* PRI refs never have a spill slot. */
289 setgcrefi(o
->gcr
, *sps
);
290 setitype(o
, irt_toitype(t
));
292 } else { /* Restore from register. */
293 Reg r
= regsp_reg(rs
);
294 lua_assert(ra_hasreg(r
));
295 if (irt_isinteger(t
)) {
296 setintV(o
, ex
->gpr
[r
-RID_MIN_GPR
]);
297 } else if (irt_isnum(t
)) {
298 setnumV(o
, ex
->fpr
[r
-RID_MIN_FPR
]);
300 } else if (irt_islightud(t
)) {
301 /* 64 bit lightuserdata which may escape already has the tag bits. */
302 o
->u64
= ex
->gpr
[r
-RID_MIN_GPR
];
306 setgcrefi(o
->gcr
, ex
->gpr
[r
-RID_MIN_GPR
]);
307 setitype(o
, irt_toitype(t
));
312 switch (bc_op(*pc
)) {
313 case BC_CALLM
: case BC_CALLMT
: case BC_RETM
: case BC_TSETM
:
314 L
->top
= frame
+ nslots
;
317 L
->top
= curr_topL(L
);
320 lua_assert(map
+ nent
== flinks
);