Issue #3680: Reference cycles created through a dict, set or deque iterator did not...
[python.git] / Objects / codeobject.c
blobe94b4cc85b1f5feb0f1fa09a8bc34b32b0e0018d
1 #include "Python.h"
2 #include "code.h"
3 #include "structmember.h"
5 #define NAME_CHARS \
6 "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz"
8 /* all_name_chars(s): true iff all chars in s are valid NAME_CHARS */
10 static int
11 all_name_chars(unsigned char *s)
13 static char ok_name_char[256];
14 static unsigned char *name_chars = (unsigned char *)NAME_CHARS;
16 if (ok_name_char[*name_chars] == 0) {
17 unsigned char *p;
18 for (p = name_chars; *p; p++)
19 ok_name_char[*p] = 1;
21 while (*s) {
22 if (ok_name_char[*s++] == 0)
23 return 0;
25 return 1;
28 static void
29 intern_strings(PyObject *tuple)
31 Py_ssize_t i;
33 for (i = PyTuple_GET_SIZE(tuple); --i >= 0; ) {
34 PyObject *v = PyTuple_GET_ITEM(tuple, i);
35 if (v == NULL || !PyString_CheckExact(v)) {
36 Py_FatalError("non-string found in code slot");
38 PyString_InternInPlace(&PyTuple_GET_ITEM(tuple, i));
43 PyCodeObject *
44 PyCode_New(int argcount, int nlocals, int stacksize, int flags,
45 PyObject *code, PyObject *consts, PyObject *names,
46 PyObject *varnames, PyObject *freevars, PyObject *cellvars,
47 PyObject *filename, PyObject *name, int firstlineno,
48 PyObject *lnotab)
50 PyCodeObject *co;
51 Py_ssize_t i;
52 /* Check argument types */
53 if (argcount < 0 || nlocals < 0 ||
54 code == NULL ||
55 consts == NULL || !PyTuple_Check(consts) ||
56 names == NULL || !PyTuple_Check(names) ||
57 varnames == NULL || !PyTuple_Check(varnames) ||
58 freevars == NULL || !PyTuple_Check(freevars) ||
59 cellvars == NULL || !PyTuple_Check(cellvars) ||
60 name == NULL || !PyString_Check(name) ||
61 filename == NULL || !PyString_Check(filename) ||
62 lnotab == NULL || !PyString_Check(lnotab) ||
63 !PyObject_CheckReadBuffer(code)) {
64 PyErr_BadInternalCall();
65 return NULL;
67 intern_strings(names);
68 intern_strings(varnames);
69 intern_strings(freevars);
70 intern_strings(cellvars);
71 /* Intern selected string constants */
72 for (i = PyTuple_Size(consts); --i >= 0; ) {
73 PyObject *v = PyTuple_GetItem(consts, i);
74 if (!PyString_Check(v))
75 continue;
76 if (!all_name_chars((unsigned char *)PyString_AS_STRING(v)))
77 continue;
78 PyString_InternInPlace(&PyTuple_GET_ITEM(consts, i));
80 co = PyObject_NEW(PyCodeObject, &PyCode_Type);
81 if (co != NULL) {
82 co->co_argcount = argcount;
83 co->co_nlocals = nlocals;
84 co->co_stacksize = stacksize;
85 co->co_flags = flags;
86 Py_INCREF(code);
87 co->co_code = code;
88 Py_INCREF(consts);
89 co->co_consts = consts;
90 Py_INCREF(names);
91 co->co_names = names;
92 Py_INCREF(varnames);
93 co->co_varnames = varnames;
94 Py_INCREF(freevars);
95 co->co_freevars = freevars;
96 Py_INCREF(cellvars);
97 co->co_cellvars = cellvars;
98 Py_INCREF(filename);
99 co->co_filename = filename;
100 Py_INCREF(name);
101 co->co_name = name;
102 co->co_firstlineno = firstlineno;
103 Py_INCREF(lnotab);
104 co->co_lnotab = lnotab;
105 co->co_zombieframe = NULL;
107 return co;
111 #define OFF(x) offsetof(PyCodeObject, x)
113 static PyMemberDef code_memberlist[] = {
114 {"co_argcount", T_INT, OFF(co_argcount), READONLY},
115 {"co_nlocals", T_INT, OFF(co_nlocals), READONLY},
116 {"co_stacksize",T_INT, OFF(co_stacksize), READONLY},
117 {"co_flags", T_INT, OFF(co_flags), READONLY},
118 {"co_code", T_OBJECT, OFF(co_code), READONLY},
119 {"co_consts", T_OBJECT, OFF(co_consts), READONLY},
120 {"co_names", T_OBJECT, OFF(co_names), READONLY},
121 {"co_varnames", T_OBJECT, OFF(co_varnames), READONLY},
122 {"co_freevars", T_OBJECT, OFF(co_freevars), READONLY},
123 {"co_cellvars", T_OBJECT, OFF(co_cellvars), READONLY},
124 {"co_filename", T_OBJECT, OFF(co_filename), READONLY},
125 {"co_name", T_OBJECT, OFF(co_name), READONLY},
126 {"co_firstlineno", T_INT, OFF(co_firstlineno), READONLY},
127 {"co_lnotab", T_OBJECT, OFF(co_lnotab), READONLY},
128 {NULL} /* Sentinel */
131 /* Helper for code_new: return a shallow copy of a tuple that is
132 guaranteed to contain exact strings, by converting string subclasses
133 to exact strings and complaining if a non-string is found. */
134 static PyObject*
135 validate_and_copy_tuple(PyObject *tup)
137 PyObject *newtuple;
138 PyObject *item;
139 Py_ssize_t i, len;
141 len = PyTuple_GET_SIZE(tup);
142 newtuple = PyTuple_New(len);
143 if (newtuple == NULL)
144 return NULL;
146 for (i = 0; i < len; i++) {
147 item = PyTuple_GET_ITEM(tup, i);
148 if (PyString_CheckExact(item)) {
149 Py_INCREF(item);
151 else if (!PyString_Check(item)) {
152 PyErr_Format(
153 PyExc_TypeError,
154 "name tuples must contain only "
155 "strings, not '%.500s'",
156 item->ob_type->tp_name);
157 Py_DECREF(newtuple);
158 return NULL;
160 else {
161 item = PyString_FromStringAndSize(
162 PyString_AS_STRING(item),
163 PyString_GET_SIZE(item));
164 if (item == NULL) {
165 Py_DECREF(newtuple);
166 return NULL;
169 PyTuple_SET_ITEM(newtuple, i, item);
172 return newtuple;
175 PyDoc_STRVAR(code_doc,
176 "code(argcount, nlocals, stacksize, flags, codestring, constants, names,\n\
177 varnames, filename, name, firstlineno, lnotab[, freevars[, cellvars]])\n\
179 Create a code object. Not for the faint of heart.");
181 static PyObject *
182 code_new(PyTypeObject *type, PyObject *args, PyObject *kw)
184 int argcount;
185 int nlocals;
186 int stacksize;
187 int flags;
188 PyObject *co = NULL;
189 PyObject *code;
190 PyObject *consts;
191 PyObject *names, *ournames = NULL;
192 PyObject *varnames, *ourvarnames = NULL;
193 PyObject *freevars = NULL, *ourfreevars = NULL;
194 PyObject *cellvars = NULL, *ourcellvars = NULL;
195 PyObject *filename;
196 PyObject *name;
197 int firstlineno;
198 PyObject *lnotab;
200 if (!PyArg_ParseTuple(args, "iiiiSO!O!O!SSiS|O!O!:code",
201 &argcount, &nlocals, &stacksize, &flags,
202 &code,
203 &PyTuple_Type, &consts,
204 &PyTuple_Type, &names,
205 &PyTuple_Type, &varnames,
206 &filename, &name,
207 &firstlineno, &lnotab,
208 &PyTuple_Type, &freevars,
209 &PyTuple_Type, &cellvars))
210 return NULL;
212 if (argcount < 0) {
213 PyErr_SetString(
214 PyExc_ValueError,
215 "code: argcount must not be negative");
216 goto cleanup;
219 if (nlocals < 0) {
220 PyErr_SetString(
221 PyExc_ValueError,
222 "code: nlocals must not be negative");
223 goto cleanup;
226 ournames = validate_and_copy_tuple(names);
227 if (ournames == NULL)
228 goto cleanup;
229 ourvarnames = validate_and_copy_tuple(varnames);
230 if (ourvarnames == NULL)
231 goto cleanup;
232 if (freevars)
233 ourfreevars = validate_and_copy_tuple(freevars);
234 else
235 ourfreevars = PyTuple_New(0);
236 if (ourfreevars == NULL)
237 goto cleanup;
238 if (cellvars)
239 ourcellvars = validate_and_copy_tuple(cellvars);
240 else
241 ourcellvars = PyTuple_New(0);
242 if (ourcellvars == NULL)
243 goto cleanup;
245 co = (PyObject *)PyCode_New(argcount, nlocals, stacksize, flags,
246 code, consts, ournames, ourvarnames,
247 ourfreevars, ourcellvars, filename,
248 name, firstlineno, lnotab);
249 cleanup:
250 Py_XDECREF(ournames);
251 Py_XDECREF(ourvarnames);
252 Py_XDECREF(ourfreevars);
253 Py_XDECREF(ourcellvars);
254 return co;
257 static void
258 code_dealloc(PyCodeObject *co)
260 Py_XDECREF(co->co_code);
261 Py_XDECREF(co->co_consts);
262 Py_XDECREF(co->co_names);
263 Py_XDECREF(co->co_varnames);
264 Py_XDECREF(co->co_freevars);
265 Py_XDECREF(co->co_cellvars);
266 Py_XDECREF(co->co_filename);
267 Py_XDECREF(co->co_name);
268 Py_XDECREF(co->co_lnotab);
269 if (co->co_zombieframe != NULL)
270 PyObject_GC_Del(co->co_zombieframe);
271 PyObject_DEL(co);
274 static PyObject *
275 code_repr(PyCodeObject *co)
277 char buf[500];
278 int lineno = -1;
279 char *filename = "???";
280 char *name = "???";
282 if (co->co_firstlineno != 0)
283 lineno = co->co_firstlineno;
284 if (co->co_filename && PyString_Check(co->co_filename))
285 filename = PyString_AS_STRING(co->co_filename);
286 if (co->co_name && PyString_Check(co->co_name))
287 name = PyString_AS_STRING(co->co_name);
288 PyOS_snprintf(buf, sizeof(buf),
289 "<code object %.100s at %p, file \"%.300s\", line %d>",
290 name, co, filename, lineno);
291 return PyString_FromString(buf);
294 static int
295 code_compare(PyCodeObject *co, PyCodeObject *cp)
297 int cmp;
298 cmp = PyObject_Compare(co->co_name, cp->co_name);
299 if (cmp) return cmp;
300 cmp = co->co_argcount - cp->co_argcount;
301 if (cmp) goto normalize;
302 cmp = co->co_nlocals - cp->co_nlocals;
303 if (cmp) goto normalize;
304 cmp = co->co_flags - cp->co_flags;
305 if (cmp) goto normalize;
306 cmp = co->co_firstlineno - cp->co_firstlineno;
307 if (cmp) goto normalize;
308 cmp = PyObject_Compare(co->co_code, cp->co_code);
309 if (cmp) return cmp;
310 cmp = PyObject_Compare(co->co_consts, cp->co_consts);
311 if (cmp) return cmp;
312 cmp = PyObject_Compare(co->co_names, cp->co_names);
313 if (cmp) return cmp;
314 cmp = PyObject_Compare(co->co_varnames, cp->co_varnames);
315 if (cmp) return cmp;
316 cmp = PyObject_Compare(co->co_freevars, cp->co_freevars);
317 if (cmp) return cmp;
318 cmp = PyObject_Compare(co->co_cellvars, cp->co_cellvars);
319 return cmp;
321 normalize:
322 if (cmp > 0)
323 return 1;
324 else if (cmp < 0)
325 return -1;
326 else
327 return 0;
330 static PyObject *
331 code_richcompare(PyObject *self, PyObject *other, int op)
333 PyCodeObject *co, *cp;
334 int eq;
335 PyObject *res;
337 if ((op != Py_EQ && op != Py_NE) ||
338 !PyCode_Check(self) ||
339 !PyCode_Check(other)) {
341 /* Py3K warning if types are not equal and comparison
342 isn't == or != */
343 if (PyErr_WarnPy3k("code inequality comparisons not supported "
344 "in 3.x", 1) < 0) {
345 return NULL;
348 Py_INCREF(Py_NotImplemented);
349 return Py_NotImplemented;
352 co = (PyCodeObject *)self;
353 cp = (PyCodeObject *)other;
355 eq = PyObject_RichCompareBool(co->co_name, cp->co_name, Py_EQ);
356 if (eq <= 0) goto unequal;
357 eq = co->co_argcount == cp->co_argcount;
358 if (!eq) goto unequal;
359 eq = co->co_nlocals == cp->co_nlocals;
360 if (!eq) goto unequal;
361 eq = co->co_flags == cp->co_flags;
362 if (!eq) goto unequal;
363 eq = co->co_firstlineno == cp->co_firstlineno;
364 if (!eq) goto unequal;
365 eq = PyObject_RichCompareBool(co->co_code, cp->co_code, Py_EQ);
366 if (eq <= 0) goto unequal;
367 eq = PyObject_RichCompareBool(co->co_consts, cp->co_consts, Py_EQ);
368 if (eq <= 0) goto unequal;
369 eq = PyObject_RichCompareBool(co->co_names, cp->co_names, Py_EQ);
370 if (eq <= 0) goto unequal;
371 eq = PyObject_RichCompareBool(co->co_varnames, cp->co_varnames, Py_EQ);
372 if (eq <= 0) goto unequal;
373 eq = PyObject_RichCompareBool(co->co_freevars, cp->co_freevars, Py_EQ);
374 if (eq <= 0) goto unequal;
375 eq = PyObject_RichCompareBool(co->co_cellvars, cp->co_cellvars, Py_EQ);
376 if (eq <= 0) goto unequal;
378 if (op == Py_EQ)
379 res = Py_True;
380 else
381 res = Py_False;
382 goto done;
384 unequal:
385 if (eq < 0)
386 return NULL;
387 if (op == Py_NE)
388 res = Py_True;
389 else
390 res = Py_False;
392 done:
393 Py_INCREF(res);
394 return res;
397 static long
398 code_hash(PyCodeObject *co)
400 long h, h0, h1, h2, h3, h4, h5, h6;
401 h0 = PyObject_Hash(co->co_name);
402 if (h0 == -1) return -1;
403 h1 = PyObject_Hash(co->co_code);
404 if (h1 == -1) return -1;
405 h2 = PyObject_Hash(co->co_consts);
406 if (h2 == -1) return -1;
407 h3 = PyObject_Hash(co->co_names);
408 if (h3 == -1) return -1;
409 h4 = PyObject_Hash(co->co_varnames);
410 if (h4 == -1) return -1;
411 h5 = PyObject_Hash(co->co_freevars);
412 if (h5 == -1) return -1;
413 h6 = PyObject_Hash(co->co_cellvars);
414 if (h6 == -1) return -1;
415 h = h0 ^ h1 ^ h2 ^ h3 ^ h4 ^ h5 ^ h6 ^
416 co->co_argcount ^ co->co_nlocals ^ co->co_flags;
417 if (h == -1) h = -2;
418 return h;
421 /* XXX code objects need to participate in GC? */
423 PyTypeObject PyCode_Type = {
424 PyVarObject_HEAD_INIT(&PyType_Type, 0)
425 "code",
426 sizeof(PyCodeObject),
428 (destructor)code_dealloc, /* tp_dealloc */
429 0, /* tp_print */
430 0, /* tp_getattr */
431 0, /* tp_setattr */
432 (cmpfunc)code_compare, /* tp_compare */
433 (reprfunc)code_repr, /* tp_repr */
434 0, /* tp_as_number */
435 0, /* tp_as_sequence */
436 0, /* tp_as_mapping */
437 (hashfunc)code_hash, /* tp_hash */
438 0, /* tp_call */
439 0, /* tp_str */
440 PyObject_GenericGetAttr, /* tp_getattro */
441 0, /* tp_setattro */
442 0, /* tp_as_buffer */
443 Py_TPFLAGS_DEFAULT, /* tp_flags */
444 code_doc, /* tp_doc */
445 0, /* tp_traverse */
446 0, /* tp_clear */
447 code_richcompare, /* tp_richcompare */
448 0, /* tp_weaklistoffset */
449 0, /* tp_iter */
450 0, /* tp_iternext */
451 0, /* tp_methods */
452 code_memberlist, /* tp_members */
453 0, /* tp_getset */
454 0, /* tp_base */
455 0, /* tp_dict */
456 0, /* tp_descr_get */
457 0, /* tp_descr_set */
458 0, /* tp_dictoffset */
459 0, /* tp_init */
460 0, /* tp_alloc */
461 code_new, /* tp_new */
464 /* All about c_lnotab.
466 c_lnotab is an array of unsigned bytes disguised as a Python string. In -O
467 mode, SET_LINENO opcodes aren't generated, and bytecode offsets are mapped
468 to source code line #s (when needed for tracebacks) via c_lnotab instead.
469 The array is conceptually a list of
470 (bytecode offset increment, line number increment)
471 pairs. The details are important and delicate, best illustrated by example:
473 byte code offset source code line number
476 50 7
477 350 307
478 361 308
480 The first trick is that these numbers aren't stored, only the increments
481 from one row to the next (this doesn't really work, but it's a start):
483 0, 1, 6, 1, 44, 5, 300, 300, 11, 1
485 The second trick is that an unsigned byte can't hold negative values, or
486 values larger than 255, so (a) there's a deep assumption that byte code
487 offsets and their corresponding line #s both increase monotonically, and (b)
488 if at least one column jumps by more than 255 from one row to the next, more
489 than one pair is written to the table. In case #b, there's no way to know
490 from looking at the table later how many were written. That's the delicate
491 part. A user of c_lnotab desiring to find the source line number
492 corresponding to a bytecode address A should do something like this
494 lineno = addr = 0
495 for addr_incr, line_incr in c_lnotab:
496 addr += addr_incr
497 if addr > A:
498 return lineno
499 lineno += line_incr
501 In order for this to work, when the addr field increments by more than 255,
502 the line # increment in each pair generated must be 0 until the remaining addr
503 increment is < 256. So, in the example above, com_set_lineno should not (as
504 was actually done until 2.2) expand 300, 300 to 255, 255, 45, 45, but to
505 255, 0, 45, 255, 0, 45.
509 PyCode_Addr2Line(PyCodeObject *co, int addrq)
511 int size = PyString_Size(co->co_lnotab) / 2;
512 unsigned char *p = (unsigned char*)PyString_AsString(co->co_lnotab);
513 int line = co->co_firstlineno;
514 int addr = 0;
515 while (--size >= 0) {
516 addr += *p++;
517 if (addr > addrq)
518 break;
519 line += *p++;
521 return line;
525 Check whether the current instruction is at the start of a line.
529 /* The theory of SET_LINENO-less tracing.
531 In a nutshell, we use the co_lnotab field of the code object
532 to tell when execution has moved onto a different line.
534 As mentioned above, the basic idea is so set things up so
535 that
537 *instr_lb <= frame->f_lasti < *instr_ub
539 is true so long as execution does not change lines.
541 This is all fairly simple. Digging the information out of
542 co_lnotab takes some work, but is conceptually clear.
544 Somewhat harder to explain is why we don't *always* call the
545 line trace function when the above test fails.
547 Consider this code:
549 1: def f(a):
550 2: if a:
551 3: print 1
552 4: else:
553 5: print 2
555 which compiles to this:
557 2 0 LOAD_FAST 0 (a)
558 3 JUMP_IF_FALSE 9 (to 15)
559 6 POP_TOP
561 3 7 LOAD_CONST 1 (1)
562 10 PRINT_ITEM
563 11 PRINT_NEWLINE
564 12 JUMP_FORWARD 6 (to 21)
565 >> 15 POP_TOP
567 5 16 LOAD_CONST 2 (2)
568 19 PRINT_ITEM
569 20 PRINT_NEWLINE
570 >> 21 LOAD_CONST 0 (None)
571 24 RETURN_VALUE
573 If 'a' is false, execution will jump to instruction at offset
574 15 and the co_lnotab will claim that execution has moved to
575 line 3. This is at best misleading. In this case we could
576 associate the POP_TOP with line 4, but that doesn't make
577 sense in all cases (I think).
579 What we do is only call the line trace function if the co_lnotab
580 indicates we have jumped to the *start* of a line, i.e. if the
581 current instruction offset matches the offset given for the
582 start of a line by the co_lnotab.
584 This also takes care of the situation where 'a' is true.
585 Execution will jump from instruction offset 12 to offset 21.
586 Then the co_lnotab would imply that execution has moved to line
587 5, which is again misleading.
589 Why do we set f_lineno when tracing? Well, consider the code
590 above when 'a' is true. If stepping through this with 'n' in
591 pdb, you would stop at line 1 with a "call" type event, then
592 line events on lines 2 and 3, then a "return" type event -- but
593 you would be shown line 5 during this event. This is a change
594 from the behaviour in 2.2 and before, and I've found it
595 confusing in practice. By setting and using f_lineno when
596 tracing, one can report a line number different from that
597 suggested by f_lasti on this one occasion where it's desirable.
601 int
602 PyCode_CheckLineNumber(PyCodeObject* co, int lasti, PyAddrPair *bounds)
604 int size, addr, line;
605 unsigned char* p;
607 p = (unsigned char*)PyString_AS_STRING(co->co_lnotab);
608 size = PyString_GET_SIZE(co->co_lnotab) / 2;
610 addr = 0;
611 line = co->co_firstlineno;
612 assert(line > 0);
614 /* possible optimization: if f->f_lasti == instr_ub
615 (likely to be a common case) then we already know
616 instr_lb -- if we stored the matching value of p
617 somwhere we could skip the first while loop. */
619 /* see comments in compile.c for the description of
620 co_lnotab. A point to remember: increments to p
621 should come in pairs -- although we don't care about
622 the line increments here, treating them as byte
623 increments gets confusing, to say the least. */
625 bounds->ap_lower = 0;
626 while (size > 0) {
627 if (addr + *p > lasti)
628 break;
629 addr += *p++;
630 if (*p)
631 bounds->ap_lower = addr;
632 line += *p++;
633 --size;
636 /* If lasti and addr don't match exactly, we don't want to
637 change the lineno slot on the frame or execute a trace
638 function. Return -1 instead.
640 if (addr != lasti)
641 line = -1;
643 if (size > 0) {
644 while (--size >= 0) {
645 addr += *p++;
646 if (*p++)
647 break;
649 bounds->ap_upper = addr;
651 else {
652 bounds->ap_upper = INT_MAX;
655 return line;