src/lj_snap.c

   1 /*
   2 ** Snapshot handling.
   3 ** Copyright (C) 2005-2010 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5
   6 #define lj_snap_c
   7 #define LUA_CORE
   8
   9 #include "lj_obj.h"
  10
  11 #if LJ_HASJIT
  12
  13 #include "lj_gc.h"
  14 #include "lj_state.h"
  15 #include "lj_frame.h"
  16 #include "lj_ir.h"
  17 #include "lj_jit.h"
  18 #include "lj_iropt.h"
  19 #include "lj_trace.h"
  20 #include "lj_snap.h"
  21 #include "lj_target.h"
  22
  23 /* Some local macros to save typing. Undef'd at the end. */
  24 #define IR(ref)         (&J->cur.ir[(ref)])
  25
  26 /* -- Snapshot buffer allocation ------------------------------------------ */
  27
  28 /* Grow snapshot buffer. */
  29 void lj_snap_grow_buf_(jit_State *J, MSize need)
  30 {
  31   MSize maxsnap = (MSize)J->param[JIT_P_maxsnap];
  32   if (need > maxsnap)
  33     lj_trace_err(J, LJ_TRERR_SNAPOV);
  34   lj_mem_growvec(J->L, J->snapbuf, J->sizesnap, maxsnap, SnapShot);
  35   J->cur.snap = J->snapbuf;
  36 }
  37
  38 /* Grow snapshot map buffer. */
  39 void lj_snap_grow_map_(jit_State *J, MSize need)
  40 {
  41   if (need < 2*J->sizesnapmap)
  42     need = 2*J->sizesnapmap;
  43   else if (need < 64)
  44     need = 64;
  45   J->snapmapbuf = (SnapEntry *)lj_mem_realloc(J->L, J->snapmapbuf,
  46                     J->sizesnapmap*sizeof(SnapEntry), need*sizeof(SnapEntry));
  47   J->cur.snapmap = J->snapmapbuf;
  48   J->sizesnapmap = need;
  49 }
  50
  51 /* -- Snapshot generation ------------------------------------------------- */
  52
  53 /* Add all modified slots to the snapshot. */
  54 static MSize snapshot_slots(jit_State *J, SnapEntry *map, BCReg nslots)
  55 {
  56   IRRef retf = J->chain[IR_RETF];  /* Limits SLOAD restore elimination. */
  57   BCReg s;
  58   MSize n = 0;
  59   for (s = 0; s < nslots; s++) {
  60     TRef tr = J->slot[s];
  61     IRRef ref = tref_ref(tr);
  62     if (ref) {
  63       SnapEntry sn = SNAP_TR(s, tr);
  64       IRIns *ir = IR(ref);
  65       if (ir->o == IR_SLOAD && ir->op1 == s && ref > retf) {
  66         /* No need to snapshot unmodified non-inherited slots. */
  67         if (!(ir->op2 & IRSLOAD_INHERIT))
  68           continue;
  69         /* No need to restore readonly slots and unmodified non-parent slots. */
  70         if ((ir->op2 & (IRSLOAD_READONLY|IRSLOAD_PARENT)) != IRSLOAD_PARENT)
  71           sn |= SNAP_NORESTORE;
  72       }
  73       map[n++] = sn;
  74     }
  75   }
  76   return n;
  77 }
  78
  79 /* Add frame links at the end of the snapshot. */
  80 static void snapshot_framelinks(jit_State *J, SnapEntry *map)
  81 {
  82   cTValue *frame = J->L->base - 1;
  83   cTValue *lim = J->L->base - J->baseslot;
  84   MSize f = 0;
  85   map[f++] = SNAP_MKPC(J->pc);  /* The current PC is always the first entry. */
  86   while (frame > lim) {  /* Backwards traversal of all frames above base. */
  87     if (frame_islua(frame)) {
  88       map[f++] = SNAP_MKPC(frame_pc(frame));
  89       frame = frame_prevl(frame);
  90     } else if (frame_ispcall(frame)) {
  91       map[f++] = SNAP_MKFTSZ(frame_ftsz(frame));
  92       frame = frame_prevd(frame);
  93     } else if (frame_iscont(frame)) {
  94       map[f++] = SNAP_MKFTSZ(frame_ftsz(frame));
  95       map[f++] = SNAP_MKPC(frame_contpc(frame));
  96       frame = frame_prevd(frame);
  97     } else {
  98       lua_assert(0);
  99     }
 100   }
 101   lua_assert(f == (MSize)(1 + J->framedepth));
 102 }
 103
 104 /* Take a snapshot of the current stack. */
 105 static void snapshot_stack(jit_State *J, SnapShot *snap, MSize nsnapmap)
 106 {
 107   BCReg nslots = J->baseslot + J->maxslot;
 108   MSize nent;
 109   SnapEntry *p;
 110   /* Conservative estimate. */
 111   lj_snap_grow_map(J, nsnapmap + nslots + (MSize)J->framedepth+1);
 112   p = &J->cur.snapmap[nsnapmap];
 113   nent = snapshot_slots(J, p, nslots);
 114   snapshot_framelinks(J, p + nent);
 115   snap->mapofs = (uint16_t)nsnapmap;
 116   snap->ref = (IRRef1)J->cur.nins;
 117   snap->nent = (uint8_t)nent;
 118   snap->depth = (uint8_t)J->framedepth;
 119   snap->nslots = (uint8_t)nslots;
 120   snap->count = 0;
 121   J->cur.nsnapmap = (uint16_t)(nsnapmap + nent + 1 + J->framedepth);
 122 }
 123
 124 /* Add or merge a snapshot. */
 125 void lj_snap_add(jit_State *J)
 126 {
 127   MSize nsnap = J->cur.nsnap;
 128   MSize nsnapmap = J->cur.nsnapmap;
 129   /* Merge if no ins. inbetween or if requested and no guard inbetween. */
 130   if (J->mergesnap ? !irt_isguard(J->guardemit) :
 131       (nsnap > 0 && J->cur.snap[nsnap-1].ref == J->cur.nins)) {
 132     nsnapmap = J->cur.snap[--nsnap].mapofs;
 133   } else {
 134     lj_snap_grow_buf(J, nsnap+1);
 135     J->cur.nsnap = (uint16_t)(nsnap+1);
 136   }
 137   J->mergesnap = 0;
 138   J->guardemit.irt = 0;
 139   snapshot_stack(J, &J->cur.snap[nsnap], nsnapmap);
 140 }
 141
 142 /* Shrink last snapshot. */
 143 void lj_snap_shrink(jit_State *J)
 144 {
 145   BCReg nslots = J->baseslot + J->maxslot;
 146   SnapShot *snap = &J->cur.snap[J->cur.nsnap-1];
 147   SnapEntry *map = &J->cur.snapmap[snap->mapofs];
 148   MSize nent = snap->nent;
 149   lua_assert(nslots < snap->nslots);
 150   snap->nslots = (uint8_t)nslots;
 151   if (nent > 0 && snap_slot(map[nent-1]) >= nslots) {
 152     MSize s, delta, depth = snap->depth;
 153     lua_assert(depth == (MSize)J->framedepth);
 154     for (nent--; nent > 0 && snap_slot(map[nent-1]) >= nslots; nent--)
 155       ;
 156     delta = snap->nent - nent;
 157     snap->nent = (uint8_t)nent;
 158     J->cur.nsnapmap = (uint16_t)(snap->mapofs + nent + 1 + depth);
 159     map += nent;
 160     for (s = 0; s <= depth; s++)  /* Move PC + frame links down. */
 161       map[s] = map[s+delta];
 162   }
 163 }
 164
 165 /* -- Snapshot access ----------------------------------------------------- */
 166
 167 /* Initialize a Bloom Filter with all renamed refs.
 168 ** There are very few renames (often none), so the filter has
 169 ** very few bits set. This makes it suitable for negative filtering.
 170 */
 171 static BloomFilter snap_renamefilter(Trace *T, SnapNo lim)
 172 {
 173   BloomFilter rfilt = 0;
 174   IRIns *ir;
 175   for (ir = &T->ir[T->nins-1]; ir->o == IR_RENAME; ir--)
 176     if (ir->op2 <= lim)
 177       bloomset(rfilt, ir->op1);
 178   return rfilt;
 179 }
 180
 181 /* Process matching renames to find the original RegSP. */
 182 static RegSP snap_renameref(Trace *T, SnapNo lim, IRRef ref, RegSP rs)
 183 {
 184   IRIns *ir;
 185   for (ir = &T->ir[T->nins-1]; ir->o == IR_RENAME; ir--)
 186     if (ir->op1 == ref && ir->op2 <= lim)
 187       rs = ir->prev;
 188   return rs;
 189 }
 190
 191 /* Convert a snapshot into a linear slot -> RegSP map.
 192 ** Note: unused slots are not initialized!
 193 */
 194 void lj_snap_regspmap(uint16_t *rsmap, Trace *T, SnapNo snapno)
 195 {
 196   SnapShot *snap = &T->snap[snapno];
 197   MSize n, nent = snap->nent;
 198   SnapEntry *map = &T->snapmap[snap->mapofs];
 199   BloomFilter rfilt = snap_renamefilter(T, snapno);
 200   for (n = 0; n < nent; n++) {
 201     SnapEntry sn = map[n];
 202     IRRef ref = snap_ref(sn);
 203     if (!irref_isk(ref)) {
 204       IRIns *ir = &T->ir[ref];
 205       uint32_t rs = ir->prev;
 206       if (bloomtest(rfilt, ref))
 207         rs = snap_renameref(T, snapno, ref, rs);
 208       rsmap[snap_slot(sn)] = (uint16_t)rs;
 209     }
 210   }
 211 }
 212
 213 /* Restore interpreter state from exit state with the help of a snapshot. */
 214 const BCIns *lj_snap_restore(jit_State *J, void *exptr)
 215 {
 216   ExitState *ex = (ExitState *)exptr;
 217   SnapNo snapno = J->exitno;  /* For now, snapno == exitno. */
 218   Trace *T = J->trace[J->parent];
 219   SnapShot *snap = &T->snap[snapno];
 220   MSize n, nent = snap->nent;
 221   SnapEntry *map = &T->snapmap[snap->mapofs];
 222   SnapEntry *flinks = map + nent + snap->depth;
 223   int32_t ftsz0;
 224   BCReg nslots = snap->nslots;
 225   TValue *frame;
 226   BloomFilter rfilt = snap_renamefilter(T, snapno);
 227   const BCIns *pc = snap_pc(map[nent]);
 228   lua_State *L = J->L;
 229
 230   /* Set interpreter PC to the next PC to get correct error messages. */
 231   setcframe_pc(cframe_raw(L->cframe), pc+1);
 232
 233   /* Make sure the stack is big enough for the slots from the snapshot. */
 234   if (LJ_UNLIKELY(L->base + nslots > L->maxstack)) {
 235     L->top = curr_topL(L);
 236     lj_state_growstack(L, nslots - curr_proto(L)->framesize);
 237   }
 238
 239   /* Fill stack slots with data from the registers and spill slots. */
 240   frame = L->base-1;
 241   ftsz0 = frame_ftsz(frame);  /* Preserve link to previous frame in slot #0. */
 242   for (n = 0; n < nent; n++) {
 243     SnapEntry sn = map[n];
 244     IRRef ref = snap_ref(sn);
 245     BCReg s = snap_slot(sn);
 246     TValue *o = &frame[s];  /* Stack slots are relative to start frame. */
 247     IRIns *ir = &T->ir[ref];
 248     if (irref_isk(ref)) {  /* Restore constant slot. */
 249       lj_ir_kvalue(L, o, ir);
 250       if ((sn & (SNAP_CONT|SNAP_FRAME))) {
 251         /* Overwrite tag with frame link. */
 252         o->fr.tp.ftsz = s != 0 ? (int32_t)*flinks-- : ftsz0;
 253         if ((sn & SNAP_FRAME)) {
 254           GCfunc *fn = ir_kfunc(ir);
 255           if (isluafunc(fn)) {
 256             MSize framesize = funcproto(fn)->framesize;
 257             TValue *fs;
 258             L->base = ++o;
 259             if (LJ_UNLIKELY(o + framesize > L->maxstack)) {  /* Grow again? */
 260               ptrdiff_t fsave = savestack(L, frame);
 261               L->top = o;
 262               lj_state_growstack(L, framesize);
 263               frame = restorestack(L, fsave);
 264               o = L->top;
 265             }
 266             fs = o + framesize;
 267           }
 268         }
 269       }
 270     } else if (!(sn & SNAP_NORESTORE)) {
 271       IRType1 t = ir->t;
 272       RegSP rs = ir->prev;
 273       lua_assert(!(sn & (SNAP_CONT|SNAP_FRAME)));
 274       if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
 275         rs = snap_renameref(T, snapno, ref, rs);
 276       if (ra_hasspill(regsp_spill(rs))) {  /* Restore from spill slot. */
 277         int32_t *sps = &ex->spill[regsp_spill(rs)];
 278         if (irt_isinteger(t)) {
 279           setintV(o, *sps);
 280         } else if (irt_isnum(t)) {
 281           o->u64 = *(uint64_t *)sps;
 282 #if LJ_64
 283         } else if (irt_islightud(t)) {
 284           /* 64 bit lightuserdata which may escape already has the tag bits. */
 285           o->u64 = *(uint64_t *)sps;
 286 #endif
 287         } else {
 288           lua_assert(!irt_ispri(t));  /* PRI refs never have a spill slot. */
 289           setgcrefi(o->gcr, *sps);
 290           setitype(o, irt_toitype(t));
 291         }
 292       } else {  /* Restore from register. */
 293         Reg r = regsp_reg(rs);
 294         lua_assert(ra_hasreg(r));
 295         if (irt_isinteger(t)) {
 296           setintV(o, ex->gpr[r-RID_MIN_GPR]);
 297         } else if (irt_isnum(t)) {
 298           setnumV(o, ex->fpr[r-RID_MIN_FPR]);
 299 #if LJ_64
 300         } else if (irt_islightud(t)) {
 301           /* 64 bit lightuserdata which may escape already has the tag bits. */
 302           o->u64 = ex->gpr[r-RID_MIN_GPR];
 303 #endif
 304         } else {
 305           if (!irt_ispri(t))
 306             setgcrefi(o->gcr, ex->gpr[r-RID_MIN_GPR]);
 307           setitype(o, irt_toitype(t));
 308         }
 309       }
 310     }
 311   }
 312   switch (bc_op(*pc)) {
 313   case BC_CALLM: case BC_CALLMT: case BC_RETM: case BC_TSETM:
 314     L->top = frame + nslots;
 315     break;
 316   default:
 317     L->top = curr_topL(L);
 318     break;
 319   }
 320   lua_assert(map + nent == flinks);
 321   return pc;
 322 }
 323
 324 #undef IR
 325
 326 #endif