| blob | 89707f9fe6b0e11e1e8c4c62e0b99323155015c7 |
1 /*
2 * very-low-level utilities for runtime support
3 */
5 /*
6 * This software is part of the SBCL system. See the README file for
7 * more information.
8 *
9 * This software is derived from the CMU CL system, which was
10 * written at Carnegie Mellon University and released into the
11 * public domain. The software is in the public domain and is
12 * provided with absolutely no warranty. See the COPYING and CREDITS
13 * files for more information.
14 */
15 \f
16 #ifdef __ELF__
17 // Mark the object as not requiring an executable stack.
18 .section .note.GNU-stack,"",%progbits
19 #endif
21 #define LANGUAGE_ASSEMBLY
22 #include "sbcl.h"
23 #include "validate.h"
24 #include "genesis/closure.h"
25 #include "genesis/funcallable-instance.h"
26 #include "genesis/fdefn.h"
27 #include "genesis/static-symbols.h"
28 #include "genesis/symbol.h"
29 #include "genesis/thread.h"
31 /* Minimize conditionalization for different OS naming schemes.
32 *
33 * (As of sbcl-0.8.10, this seems no longer to be much of an issue,
34 * since everyone has converged on ELF. If this generality really
35 * turns out not to matter, perhaps it's just clutter we could get
36 * rid of? -- WHN 2004-04-18)
37 *
38 * (Except Win32, which is unlikely ever to be ELF, sorry. -- AB 2005-12-08)
39 */
40 #if defined __linux__ || defined LISP_FEATURE_FREEBSD || defined __NetBSD__ || defined __OpenBSD__ || \
41 defined __sun || defined __DragonFly__
42 #define GNAME(var) var
43 #else
44 #define GNAME(var) _##var
45 #endif
47 /* Get the right type of alignment. Linux, FreeBSD and NetBSD (but not OpenBSD)
48 * want alignment in bytes.
49 *
50 * (As in the GNAME() definitions above, as of sbcl-0.8.10, this seems
51 * no longer to be much of an issue, since everyone has converged on
52 * the same value. If this generality really turns out not to
53 * matter any more, perhaps it's just clutter we could get
54 * rid of? -- WHN 2004-04-18)
55 */
56 #if defined(__linux__) || defined(LISP_FEATURE_FREEBSD) || defined(__NetBSD__) || defined(__OpenBSD__) || \
57 defined(__sun) || defined(LISP_FEATURE_WIN32) || defined(__DragonFly__)
58 #define align_4byte 4
59 #define align_8byte 8
60 #define align_16byte 16
61 #define align_page 4096
62 #else
63 #define align_4byte 2
64 #define align_8byte 3
65 #define align_16byte 4
66 #define align_page 12
67 #endif
69 /*
70 * The assembler used for win32 doesn't like .type or .size directives,
71 * so we want to conditionally kill them out. So let's wrap them in macros
72 * that are defined to be no-ops on win32. Hopefully this still works on
73 * other platforms.
74 */
75 #if !defined(LISP_FEATURE_WIN32) && !defined(LISP_FEATURE_DARWIN)
76 #define TYPE(name) .type name,@function
77 #define SIZE(name) .size name,.-name
78 #else
79 #define TYPE(name)
80 #define SIZE(name)
81 #endif
83 /* Helper macros for access to thread-locals slots for both OS types:
84 * ------------------------------------------------------------------------
85 *
86 * Windows TEB block
87 * ================== __________
88 * | Win32 %FS base | ----> | | 0
89 * ================== | | 1
90 * z z
91 * TLS slots start here> |XXXXXXXX| e10 = TEB_STATIC_TLS_SLOTS_OFFSET
92 * |XXXXXXXX| e11
93 * z ... z
94 * |XXXXXXXX| e4e
95 * TLS ends here> ,- |XXXXXXXX| e4f = TEB_STATIC_TLS_SLOTS_OFFSET+63
96 * / z z
97 * | ---------- "os_address" ----.
98 * | |
99 * | big blob of SBCL-specific thread-local data |
100 * | |----------------------------------------| <--'
101 * | | CONTROL, BINDING, ALIEN STACK |
102 * | z z
103 * ================== | |----------------------------------------|
104 * | Linux %FS base | -->| | FFI stack pointer |
105 * ================== | | (extra page for mprotect) |
106 * \ |----------------------------------------|
107 * (union p_t_d) -----> \-> | struct thread { | dynamic_values[0] |
108 * . | ... | [1] |
109 * . z ... z ... z
110 * [tls data begins] | } | ... | <-
111 * [declared end of p_t_d] |----------------------------------------| . |
112 * . | ... | . |
113 * . | [TLS_SIZE-1] | <-|
114 * [tls data actually ends] |----------------------------------------| |
115 * . | ALTSTACK | |
116 * . |----------------------------------------| |
117 * . | struct nonpointer_thread_data { } | |
118 * . ------------------------------------------ |
119 * [blob actually ends] |
120 * /
121 * /
122 * /
123 * ______________________ /
124 * | struct symbol { | /
125 * z ... z /
126 * | fixnum tls_index; // fixnum value relative to union /
127 * | } | (< TLS_SIZE = 4096)
128 * ---------------------|
129 */
130 #ifdef LISP_FEATURE_WIN32
131 # define TEB_STATIC_TLS_SLOTS_OFFSET 0xE10
132 # define TEB_SBCL_THREAD_BASE_OFFSET (TEB_STATIC_TLS_SLOTS_OFFSET+(63*4))
133 # define SBCL_THREAD_BASE_EA %fs:TEB_SBCL_THREAD_BASE_OFFSET
134 # define MAYBE_FS(addr) addr
135 # define LoadTlSymbolValueAddress(symbol,reg) ; \
136 movl SBCL_THREAD_BASE_EA, reg ; \
137 addl (symbol+SYMBOL_TLS_INDEX_OFFSET), reg ;
138 # define LoadCurrentThreadSlot(offset,reg); \
139 movl SBCL_THREAD_BASE_EA, reg ; \
140 movl offset(reg), reg ;
141 #elif defined(LISP_FEATURE_LINUX) || defined(LISP_FEATURE_SUNOS) || defined(LISP_FEATURE_FREEBSD) || \
142 defined(LISP_FEATURE_DRAGONFLY)
143 /* %fs:0 refers to the current thread. Useful! Less usefully,
144 * Linux/x86 isn't capable of reporting a faulting si_addr on a
145 * segment as defined above (whereas faults on the segment that %gs
146 * usually points are reported just fine...).
147 * But we have the thread's address stored in the THIS slot,
148 * so that within the thread
149 * movl %fs:THIS_OFFSET,x
150 * stores the absolute address of %fs:0 into x.
151 */
152 # define SBCL_THREAD_BASE_EA %fs:THREAD_THIS_OFFSET
153 # define MAYBE_FS(addr) addr
154 #else
155 /* perhaps there's an OS out there that actually supports %fs without
156 * jumping through hoops, so just in case, here a default definition: */
157 # define SBCL_THREAD_BASE_EA $0
158 # define MAYBE_FS(addr) %fs:addr
159 #endif
161 /* gas can't parse 4096LU; redefine */
162 #if BACKEND_PAGE_BYTES == 4096
163 # undef BACKEND_PAGE_BYTES
164 # define BACKEND_PAGE_BYTES 4096
165 #elif BACKEND_PAGE_BYTES == 32768
166 # undef BACKEND_PAGE_BYTES
167 # define BACKEND_PAGE_BYTES 32768
168 #else
169 # error BACKEND_PAGE_BYTES mismatch
170 #endif
172 /* OAOOM because we don't have the C headers here */
173 #define THREAD_CSP_PAGE_SIZE BACKEND_PAGE_BYTES
175 /* the CSP page sits right before the thread */
176 #define THREAD_SAVED_CSP_OFFSET (-THREAD_CSP_PAGE_SIZE)
178 /*
179 * x86/darwin (as of MacOS X 10.4.5) doesn't reliably file signal
180 * handlers (SIGTRAP or Mach exception handlers) for 0xCC, wo we have
181 * to use ud2 instead. ud2 is an undefined opcode, #x0b0f, or
182 * 0F 0B in low-endian notation, that causes SIGILL to fire. We check
183 * for this instruction in the SIGILL handler and if we see it, we
184 * advance the EIP by two bytes to skip over ud2 instruction and
185 * call sigtrap_handler. */
186 #if defined(LISP_FEATURE_UD2_BREAKPOINTS)
187 #define TRAP ud2
188 #else
189 #define TRAP int3
190 #endif
192 .text
193 .globl GNAME(all_threads)
194 \f
195 /*
196 * A call to call_into_c preserves esi, edi, and ebp.
197 * (The C function will preserve ebx, esi, edi, and ebp across its
198 * function call, but we trash ebx ourselves by using it to save the
199 * return Lisp address.)
200 *
201 * Return values are in eax and maybe edx for quads, or st(0) for
202 * floats.
203 *
204 * This should work for Lisp calls C calls Lisp calls C..
205 *
206 * FIXME & OAOOM: This duplicates call-out in src/compiler/x86/c-call.lisp,
207 * so if you tweak this, change that too!
208 */
209 /*
210 * Note on sections specific to LISP_FEATURE_SB_SAFEPOINT:
211 *
212 * The code below is essential to safepoint-based garbage collection,
213 * and several details need to be considered for correct implementation.
214 *
215 * The stack spilling approach:
216 * On SB-SAFEPOINT platforms, the CALL-OUT vop is defined to spill all
217 * live Lisp TNs to the stack to provide information for conservative
218 * GC cooperatively (avoiding the need to retrieve register values
219 * from POSIX signal contexts or Windows GetThreadContext()).
220 *
221 * Finding the SP at all:
222 * The main remaining value needed by GC is the stack pointer (SP) at
223 * the moment of entering the foreign function. For this purpose, a
224 * thread-local field for the SP is used. Two stores to that field
225 * are done for each C call, one to save the SP before calling out and
226 * and one to undo that store afterwards.
227 *
228 * Stores as synchronization points:
229 * These two stores delimit the C call: While the SP is set, our
230 * thread is known not to run Lisp code: During GC, memory protection
231 * ensures that no thread proceeds across stores.
232 *
233 * The return PC issue:
234 * (Note that CALL-OUT has, in principle, two versions: Inline
235 * assembly in the VOP -or- alternatively the out-of-line version you
236 * are currently reading. In reality, safepoint builds currently
237 * lack the inline code entirely.)
238 *
239 * Both versions need to take special care with the return PC:
240 * - In the inline version of the code (if it existed), the two stores
241 * would be done directly in the CALL-OUT vop. In that theoretical
242 * implementation, there is a time interval between return of the
243 * actual C call and a second SP store during which the return
244 * address might not be on the stack anymore.
245 * - In this out-of-line version, the stores are done during
246 * call_into_c's frame, but an equivalent problem arises: In order
247 * to present the stack of arguments as our foreign function expects
248 * them, call_into_c has to pop the Lisp return address into a
249 * register first; this register has to be preserved by GENCGC
250 * separately: our return address is not in the stack anymore.
251 * In both case, stack scanning alone is not sufficient to pin
252 * the return address, and we communicate it to GC explicitly
253 * in addition to the SP.
254 *
255 * Note on look-alike accessor macros with vastly different behaviour:
256 * THREAD_PC_AROUND_FOREIGN_CALL_OFFSET is an "ordinary" field of the
257 * struct thread, whereas THREAD_SAVED_CSP_OFFSET is a synchronization
258 * point on a potentially write-protected page.
259 */
261 .text
262 .align align_16byte,0x90
263 .globl GNAME(call_into_c)
264 TYPE(GNAME(call_into_c))
265 GNAME(call_into_c):
266 /* Save the return Lisp address in ebx. */
267 popl %ebx
269 /* Setup the NPX for C */
270 /* The VOP says regarding CLD: "Clear out DF: Darwin, Windows,
271 * and Solaris at least require this, and it should not hurt
272 * others either." call_into_c didn't have it, but better safe than
273 * sorry. */
274 cld
275 fstp %st(0)
276 fstp %st(0)
277 fstp %st(0)
278 fstp %st(0)
279 fstp %st(0)
280 fstp %st(0)
281 fstp %st(0)
282 fstp %st(0)
284 #ifdef LISP_FEATURE_SB_SAFEPOINT
285 /* enter safe region: store SP and return PC */
286 movl SBCL_THREAD_BASE_EA,%edi
287 movl %esp,MAYBE_FS(THREAD_SAVED_CSP_OFFSET(%edi))
288 movl %ebx,MAYBE_FS(THREAD_PC_AROUND_FOREIGN_CALL_OFFSET(%edi))
289 #endif
291 /* foreign call, preserving ESI, EDI, and EBX */
292 call *%eax # normal callout using Lisp stack
293 /* return values now in eax/edx OR st(0) */
295 #ifdef LISP_FEATURE_SB_SAFEPOINT
296 /* leave region: clear the SP! (Also unpin the return PC.) */
297 xorl %ecx,%ecx
298 movl %ecx,MAYBE_FS(THREAD_SAVED_CSP_OFFSET(%edi))
299 movl %ecx,MAYBE_FS(THREAD_PC_AROUND_FOREIGN_CALL_OFFSET(%edi))
300 #endif
302 movl %eax,%ecx # remember integer return value
304 /* Check for a return FP value. */
305 fxam
306 fnstsw %ax
307 andl $0x4500,%eax
308 cmpl $0x4100,%eax
309 jne Lfp_rtn_value
311 /* The return value is in eax, or eax,edx? */
312 /* Set up the NPX stack for Lisp. */
313 fldz # Ensure no regs are empty.
314 fldz
315 fldz
316 fldz
317 fldz
318 fldz
319 fldz
320 fldz
322 /* Restore the return value. */
323 movl %ecx,%eax # maybe return value
325 /* Return. */
326 jmp *%ebx
328 Lfp_rtn_value:
329 /* The return result is in st(0). */
330 /* Set up the NPX stack for Lisp, placing the result in st(0). */
331 fldz # Ensure no regs are empty.
332 fldz
333 fldz
334 fldz
335 fldz
336 fldz
337 fldz
338 fxch %st(7) # Move the result back to st(0).
340 /* We don't need to restore eax, because the result is in st(0). */
342 /* Return. FIXME: It would be nice to restructure this to use RET. */
343 jmp *%ebx
345 SIZE(GNAME(call_into_c))
347 \f
348 .text
349 .globl GNAME(call_into_lisp_first_time)
350 TYPE(GNAME(call_into_lisp_first_time))
352 /* We don't worry too much about saving registers
353 * here, because we never expect to return from the initial call to lisp
354 * anyway */
356 .align align_16byte,0x90
357 GNAME(call_into_lisp_first_time):
358 pushl %ebp # Save old frame pointer.
359 movl %esp,%ebp # Establish new frame.
360 #ifndef LISP_FEATURE_WIN32
361 movl GNAME(all_threads),%eax
362 /* pthread machinery takes care of this for other threads */
363 movl THREAD_CONTROL_STACK_END_OFFSET(%eax) ,%esp
364 #else
365 /* Win32 -really- doesn't like you switching stacks out from under it. */
366 movl GNAME(all_threads),%eax
367 #endif
368 jmp Lstack
369 \f
370 .text
371 .globl GNAME(call_into_lisp)
372 TYPE(GNAME(call_into_lisp))
374 /* The C conventions require that ebx, esi, edi, and ebp be preserved
375 * across function calls. */
377 .align align_16byte,0x90
378 GNAME(call_into_lisp):
379 pushl %ebp # Save old frame pointer.
380 movl %esp,%ebp # Establish new frame.
382 Lstack:
383 /* Save the NPX state */
384 fwait # Catch any pending NPX exceptions.
385 subl $108,%esp # Make room for the NPX state.
386 fnsave (%esp) # save and reset NPX
388 movl (%esp),%eax # Load NPX control word.
389 andl $0xfffff2ff,%eax # Set rounding mode to nearest.
390 orl $0x00000200,%eax # Set precision to 64 bits. (53-bit mantissa)
391 pushl %eax
392 fldcw (%esp) # Recover modes.
393 popl %eax
395 fldz # Ensure no FP regs are empty.
396 fldz
397 fldz
398 fldz
399 fldz
400 fldz
401 fldz
402 fldz
404 /* Save C regs: ebx esi edi. */
405 pushl %ebx
406 pushl %esi
407 pushl %edi
409 /* Clear descriptor regs. */
410 xorl %eax,%eax # lexenv
411 xorl %ebx,%ebx # available
412 xorl %ecx,%ecx # arg count
413 xorl %edx,%edx # first arg
414 xorl %edi,%edi # second arg
415 xorl %esi,%esi # third arg
417 /* no longer in function call */
418 movl %esp,%ebx # remember current stack
419 pushl %ebx # Save entry stack on (maybe) new stack.
421 /* Establish Lisp args. */
422 movl 8(%ebp),%eax # lexenv?
423 movl 12(%ebp),%ebx # address of arg vec
424 movl 16(%ebp),%ecx # num args
425 shll $2,%ecx # Make num args into fixnum.
426 cmpl $0,%ecx
427 je Ldone
428 movl (%ebx),%edx # arg0
429 cmpl $4,%ecx
430 je Ldone
431 movl 4(%ebx),%edi # arg1
432 cmpl $8,%ecx
433 je Ldone
434 movl 8(%ebx),%esi # arg2
435 Ldone:
436 /* Registers eax, ecx, edx, edi, and esi are now live. */
438 #ifdef LISP_FEATURE_WIN32
439 /* Establish an SEH frame. */
440 #ifdef LISP_FEATURE_SB_THREAD
441 /* Save binding stack pointer */
442 subl $4, %esp
443 pushl %eax
444 movl SBCL_THREAD_BASE_EA, %eax
445 movl THREAD_BINDING_STACK_POINTER_OFFSET(%eax), %eax
446 movl %eax, 4(%esp)
447 popl %eax
448 #else
449 pushl BINDING_STACK_POINTER + SYMBOL_VALUE_OFFSET
450 #endif
451 pushl $GNAME(exception_handler_wrapper)
452 pushl %fs:0
453 movl %esp, %fs:0
454 #endif
456 /* Alloc new frame. */
457 push %ebp # Dummy for return address
458 push %ebp # fp in save location S1
459 mov %esp,%ebp # The current sp marks start of new frame.
460 sub $4,%esp # Ensure 3 slots are allocated, two above.
462 call *CLOSURE_FUN_OFFSET(%eax)
464 /* If the function returned multiple values, it will return to
465 this point. Lose them */
466 jnc LsingleValue
467 mov %ebx, %esp
468 LsingleValue:
469 /* A singled value function returns here */
471 #ifdef LISP_FEATURE_WIN32
472 /* Remove our SEH frame. */
473 mov %fs:0,%esp
474 popl %fs:0
475 add $8, %esp
476 #endif
478 /* Restore the stack, in case there was a stack change. */
479 popl %esp # c-sp
481 /* Restore C regs: ebx esi edi. */
482 popl %edi
483 popl %esi
484 popl %ebx
486 /* Restore the NPX state. */
487 frstor (%esp)
488 addl $108, %esp
490 popl %ebp # c-sp
491 movl %edx,%eax # c-val
492 ret
493 SIZE(GNAME(call_into_lisp))
494 \f
495 /* support for saving and restoring the NPX state from C */
496 .text
497 .globl GNAME(fpu_save)
498 TYPE(GNAME(fpu_save))
499 .align 2,0x90
500 GNAME(fpu_save):
501 movl 4(%esp),%eax
502 fnsave (%eax) # Save the NPX state. (resets NPX)
503 ret
504 SIZE(GNAME(fpu_save))
506 .globl GNAME(fpu_restore)
507 TYPE(GNAME(fpu_restore))
508 .align 2,0x90
509 GNAME(fpu_restore):
510 movl 4(%esp),%eax
511 frstor (%eax) # Restore the NPX state.
512 ret
513 SIZE(GNAME(fpu_restore))
514 \f
515 /*
516 * fun-end breakpoint magic
517 */
519 /*
520 * For an explanation of the magic involved in function-end
521 * breakpoints, see the implementation in ppc-assem.S.
522 */
524 .text
525 .globl GNAME(fun_end_breakpoint_guts)
526 .align align_16byte
527 GNAME(fun_end_breakpoint_guts):
528 /* Multiple Value return */
529 jc multiple_value_return
530 /* Single value return: The eventual return will now use the
531 multiple values return convention but with a return values
532 count of one. */
533 movl %esp,%ebx # Setup ebx - the ofp.
534 subl $4,%esp # Allocate one stack slot for the return value
535 movl $(1 << N_FIXNUM_TAG_BITS),%ecx # Setup ecx for one return value.
536 movl $(NIL),%edi # default second value
537 movl $(NIL),%esi # default third value
539 multiple_value_return:
541 .globl GNAME(fun_end_breakpoint_trap)
542 GNAME(fun_end_breakpoint_trap):
543 TRAP
544 .byte trap_FunEndBreakpoint
545 hlt # We should never return here.
547 .globl GNAME(fun_end_breakpoint_end)
548 GNAME(fun_end_breakpoint_end):
550 \f
551 .globl GNAME(do_pending_interrupt)
552 TYPE(GNAME(do_pending_interrupt))
553 .align align_16byte,0x90
554 GNAME(do_pending_interrupt):
555 TRAP
556 .byte trap_PendingInterrupt
557 ret
558 SIZE(GNAME(do_pending_interrupt))
559 \f
560 /* Allocate bytes and return the start of the allocated space
561 * in the specified destination register.
562 *
563 * In the general case the size will be in the destination register.
564 *
565 * All registers must be preserved except the destination.
566 * The C conventions will preserve ebx, esi, edi, and ebp.
567 * So only eax, ecx, and edx need special care here.
568 *
569 * ALLOC factors out the logic of calling alloc(): stack alignment, etc.
570 *
571 * DEFINE_ALLOC_TO_FOO defines an alloction routine.
572 */
574 #ifdef LISP_FEATURE_DARWIN
575 #define ALLOC(size) \
576 pushl %ebp; /* Save EBP */ \
577 movl %esp,%ebp; /* Save ESP to EBP */ \
578 pushl $0; /* Reserve space for arg */ \
579 andl $0xfffffff0,%esp; /* Align stack to 16bytes */ \
580 movl size, (%esp); /* Argument to alloc */ \
581 call GNAME(alloc); \
582 movl %ebp,%esp; /* Restore ESP from EBP */ \
583 popl %ebp; /* Restore EBP */
584 #else
585 #define ALLOC(size) \
586 pushl size; /* Argument to alloc */ \
587 call GNAME(alloc); \
588 addl $4,%esp; /* Pop argument */
589 #endif
591 #define DEFINE_ALLOC_TO_EAX(name,size) \
592 .globl GNAME(name); \
593 TYPE(GNAME(name)); \
594 .align align_16byte,0x90; \
595 GNAME(name): \
596 pushl %ecx; /* Save ECX and EDX */ \
597 pushl %edx; \
598 ALLOC(size) \
599 popl %edx; /* Restore ECX and EDX */ \
600 popl %ecx; \
601 ret; \
602 SIZE(GNAME(name))
604 #define DEFINE_ALLOC_TO_ECX(name,size) \
605 .globl GNAME(name); \
606 TYPE(GNAME(name)); \
607 .align align_16byte,0x90; \
608 GNAME(name): \
609 pushl %eax; /* Save EAX and EDX */ \
610 pushl %edx; \
611 ALLOC(size) \
612 movl %eax,%ecx; /* Result to destination */ \
613 popl %edx; \
614 popl %eax; \
615 ret; \
616 SIZE(GNAME(name))
618 #define DEFINE_ALLOC_TO_EDX(name,size) \
619 .globl GNAME(name); \
620 TYPE(GNAME(name)); \
621 .align align_16byte,0x90; \
622 GNAME(name): \
623 pushl %eax; /* Save EAX and ECX */ \
624 pushl %ecx; \
625 ALLOC(size) \
626 movl %eax,%edx; /* Restore EAX and ECX */ \
627 popl %ecx; \
628 popl %eax; \
629 ret; \
630 SIZE(GNAME(name))
632 #define DEFINE_ALLOC_TO_REG(name,reg,size) \
633 .globl GNAME(name); \
634 TYPE(GNAME(name)); \
635 .align align_16byte,0x90; \
636 GNAME(name): \
637 pushl %eax; /* Save EAX, ECX, and EDX */ \
638 pushl %ecx; \
639 pushl %edx; \
640 ALLOC(size) \
641 movl %eax,reg; /* Restore them */ \
642 popl %edx; \
643 popl %ecx; \
644 popl %eax; \
645 ret; \
646 SIZE(GNAME(name))
648 DEFINE_ALLOC_TO_EAX(alloc_to_eax,%eax)
649 DEFINE_ALLOC_TO_EAX(alloc_8_to_eax,$8)
650 DEFINE_ALLOC_TO_EAX(alloc_16_to_eax,$16)
652 DEFINE_ALLOC_TO_ECX(alloc_to_ecx,%ecx)
653 DEFINE_ALLOC_TO_ECX(alloc_8_to_ecx,$8)
654 DEFINE_ALLOC_TO_ECX(alloc_16_to_ecx,$16)
656 DEFINE_ALLOC_TO_EDX(alloc_to_edx,%edx)
657 DEFINE_ALLOC_TO_EDX(alloc_8_to_edx,$8)
658 DEFINE_ALLOC_TO_EDX(alloc_16_to_edx,$16)
660 DEFINE_ALLOC_TO_REG(alloc_to_ebx,%ebx,%ebx)
661 DEFINE_ALLOC_TO_REG(alloc_8_to_ebx,%ebx,$8)
662 DEFINE_ALLOC_TO_REG(alloc_16_to_ebx,%ebx,$16)
664 DEFINE_ALLOC_TO_REG(alloc_to_esi,%esi,%esi)
665 DEFINE_ALLOC_TO_REG(alloc_8_to_esi,%esi,$8)
666 DEFINE_ALLOC_TO_REG(alloc_16_to_esi,%esi,$16)
668 DEFINE_ALLOC_TO_REG(alloc_to_edi,%edi,%edi)
669 DEFINE_ALLOC_TO_REG(alloc_8_to_edi,%edi,$8)
670 DEFINE_ALLOC_TO_REG(alloc_16_to_edi,%edi,$16)
672 /* Called from lisp when an inline allocation overflows.
673 * Every register except the result needs to be preserved.
674 * We depend on C to preserve ebx, esi, edi, and ebp.
675 * But where necessary must save eax, ecx, edx. */
677 #ifdef LISP_FEATURE_SB_THREAD
678 #define START_REGION %fs:THREAD_ALLOC_REGION_OFFSET
679 #else
680 #define START_REGION GNAME(boxed_region)
681 #endif
683 #if defined(LISP_FEATURE_SB_THREAD) && defined(LISP_FEATURE_WIN32)
684 #define ALLOC_OVERFLOW(size,scratch) \
685 movl SBCL_THREAD_BASE_EA, scratch; \
686 /* Calculate the size for the allocation. */ \
687 subl THREAD_ALLOC_REGION_OFFSET(scratch),size; \
688 ALLOC(size)
689 #else
690 #define ALLOC_OVERFLOW(size,scratch) \
691 /* Calculate the size for the allocation. */ \
692 subl START_REGION,size; \
693 ALLOC(size)
694 #endif
696 /* This routine handles an overflow with eax=crfp+size. So the
697 size=eax-crfp. */
698 .align align_16byte
699 .globl GNAME(alloc_overflow_eax)
700 TYPE(GNAME(alloc_overflow_eax))
701 GNAME(alloc_overflow_eax):
702 pushl %ecx # Save ecx
703 pushl %edx # Save edx
704 ALLOC_OVERFLOW(%eax,%edx)
705 popl %edx # Restore edx.
706 popl %ecx # Restore ecx.
707 ret
708 SIZE(GNAME(alloc_overflow_eax))
710 .align align_16byte
711 .globl GNAME(alloc_overflow_ecx)
712 TYPE(GNAME(alloc_overflow_ecx))
713 GNAME(alloc_overflow_ecx):
714 pushl %eax # Save eax
715 pushl %edx # Save edx
716 ALLOC_OVERFLOW(%ecx,%edx)
717 movl %eax,%ecx # setup the destination.
718 popl %edx # Restore edx.
719 popl %eax # Restore eax.
720 ret
721 SIZE(GNAME(alloc_overflow_ecx))
723 .align align_16byte
724 .globl GNAME(alloc_overflow_edx)
725 TYPE(GNAME(alloc_overflow_edx))
726 GNAME(alloc_overflow_edx):
727 pushl %eax # Save eax
728 pushl %ecx # Save ecx
729 ALLOC_OVERFLOW(%edx,%ecx)
730 movl %eax,%edx # setup the destination.
731 popl %ecx # Restore ecx.
732 popl %eax # Restore eax.
733 ret
734 SIZE(GNAME(alloc_overflow_edx))
736 /* This routine handles an overflow with ebx=crfp+size. So the
737 size=ebx-crfp. */
738 .align align_16byte
739 .globl GNAME(alloc_overflow_ebx)
740 TYPE(GNAME(alloc_overflow_ebx))
741 GNAME(alloc_overflow_ebx):
742 pushl %eax # Save eax
743 pushl %ecx # Save ecx
744 pushl %edx # Save edx
745 ALLOC_OVERFLOW(%ebx,%edx)
746 movl %eax,%ebx # setup the destination.
747 popl %edx # Restore edx.
748 popl %ecx # Restore ecx.
749 popl %eax # Restore eax.
750 ret
751 SIZE(GNAME(alloc_overflow_ebx))
753 /* This routine handles an overflow with esi=crfp+size. So the
754 size=esi-crfp. */
755 .align align_16byte
756 .globl GNAME(alloc_overflow_esi)
757 TYPE(GNAME(alloc_overflow_esi))
758 GNAME(alloc_overflow_esi):
759 pushl %eax # Save eax
760 pushl %ecx # Save ecx
761 pushl %edx # Save edx
762 ALLOC_OVERFLOW(%esi,%edx)
763 movl %eax,%esi # setup the destination.
764 popl %edx # Restore edx.
765 popl %ecx # Restore ecx.
766 popl %eax # Restore eax.
767 ret
768 SIZE(GNAME(alloc_overflow_esi))
770 .align align_16byte
771 .globl GNAME(alloc_overflow_edi)
772 TYPE(GNAME(alloc_overflow_edi))
773 GNAME(alloc_overflow_edi):
774 pushl %eax # Save eax
775 pushl %ecx # Save ecx
776 pushl %edx # Save edx
777 ALLOC_OVERFLOW(%edi,%edx)
778 movl %eax,%edi # setup the destination.
779 popl %edx # Restore edx.
780 popl %ecx # Restore ecx.
781 popl %eax # Restore eax.
782 ret
783 SIZE(GNAME(alloc_overflow_edi))
786 #ifdef LISP_FEATURE_WIN32
787 /* The guts of the exception-handling system doesn't use
788 * frame pointers, which manages to throw off backtraces
789 * rather badly. So here we grab the (known-good) EBP
790 * and EIP from the exception context and use it to fake
791 * up a stack frame which will skip over the system SEH
792 * code. */
793 .align align_16byte
794 .globl GNAME(exception_handler_wrapper)
795 TYPE(GNAME(exception_handler_wrapper))
796 GNAME(exception_handler_wrapper):
797 /* Context layout is: */
798 /* 7 dwords before FSA. (0x1c) */
799 /* 8 dwords and 0x50 bytes in the FSA. (0x70/0x8c) */
800 /* 4 dwords segregs. (0x10/0x9c) */
801 /* 6 dwords non-stack GPRs. (0x18/0xb4) */
802 /* EBP (at 0xb4) */
803 /* EIP (at 0xb8) */
804 #define CONTEXT_EBP_OFFSET 0xb4
805 #define CONTEXT_EIP_OFFSET 0xb8
806 /* some other stuff we don't care about. */
807 pushl %ebp
808 movl 0x10(%esp), %ebp /* context */
809 pushl CONTEXT_EIP_OFFSET(%ebp)
810 pushl CONTEXT_EBP_OFFSET(%ebp)
811 movl %esp, %ebp
812 pushl 0x1c(%esp)
813 pushl 0x1c(%esp)
814 pushl 0x1c(%esp)
815 pushl 0x1c(%esp)
816 call GNAME(handle_exception)
817 lea 8(%ebp), %esp
818 popl %ebp
819 ret
820 SIZE(GNAME(exception_handler_wrapper))
821 #endif
823 #ifdef LISP_FEATURE_DARWIN
824 .align align_16byte
825 .globl GNAME(call_into_lisp_tramp)
826 TYPE(GNAME(call_into_lisp_tramp))
827 GNAME(call_into_lisp_tramp):
828 /* 1. build the stack frame from the block that's pointed to by ECX
829 2. free the block
830 3. set ECX to 0
831 4. call the function via call_into_lisp
832 */
833 pushl 0(%ecx) /* return address */
835 pushl %ebp
836 movl %esp, %ebp
838 pushl 32(%ecx) /* eflags */
839 pushl 28(%ecx) /* EAX */
840 pushl 20(%ecx) /* ECX */
841 pushl 16(%ecx) /* EDX */
842 pushl 24(%ecx) /* EBX */
843 pushl $0 /* popal is going to ignore esp */
844 pushl %ebp /* is this right?? */
845 pushl 12(%ecx) /* ESI */
846 pushl 8(%ecx) /* EDI */
847 pushl $0 /* args for call_into_lisp */
848 pushl $0
849 pushl 4(%ecx) /* function to call */
851 /* free our save block */
852 pushl %ecx /* reserve sufficient space on stack for args */
853 pushl %ecx
854 andl $0xfffffff0, %esp /* align stack */
855 movl $0x40, 4(%esp)
856 movl %ecx, (%esp)
857 call GNAME(os_invalidate)
859 /* call call_into_lisp */
860 leal -48(%ebp), %esp
861 call GNAME(call_into_lisp)
863 /* Clean up our mess */
864 leal -36(%ebp), %esp
865 popal
866 popfl
867 leave
868 ret
870 SIZE(call_into_lisp_tramp)
871 #endif
873 .align align_16byte,0x90
874 .globl GNAME(post_signal_tramp)
875 TYPE(GNAME(post_signal_tramp))
876 GNAME(post_signal_tramp):
877 /* this is notionally the second half of a function whose first half
878 * doesn't exist. This is where call_into_lisp returns when called
879 * using return_to_lisp_function */
880 addl $12,%esp /* clear call_into_lisp args from stack */
881 popal /* restore registers */
882 popfl
883 #ifdef LISP_FEATURE_DARWIN
884 /* skip two padding words */
885 addl $8,%esp
886 #endif
887 leave
888 ret
889 SIZE(GNAME(post_signal_tramp))
892 /* fast_bzero implementations and code to detect which implementation
893 * to use.
894 */
895 \f
896 .globl GNAME(fast_bzero_pointer)
897 .data
898 .align align_16byte
899 GNAME(fast_bzero_pointer):
900 /* Variable containing a pointer to the bzero function to use.
901 * Initially points to a basic function. Change this variable
902 * to fast_bzero_detect if OS supports SSE. */
903 .long GNAME(fast_bzero_base)
904 \f
905 .text
906 .align align_16byte,0x90
907 .globl GNAME(fast_bzero)
908 TYPE(GNAME(fast_bzero))
909 GNAME(fast_bzero):
910 /* Indirect function call */
911 jmp *GNAME(fast_bzero_pointer)
912 SIZE(GNAME(fast_bzero))
914 \f
915 .text
916 .align align_16byte,0x90
917 .globl GNAME(fast_bzero_detect)
918 TYPE(GNAME(fast_bzero_detect))
919 GNAME(fast_bzero_detect):
920 /* Decide whether to use SSE, MMX or REP version */
921 push %eax /* CPUID uses EAX-EDX */
922 push %ebx
923 push %ecx
924 push %edx
925 mov $1, %eax
926 cpuid
927 test $0x04000000, %edx /* SSE2 needed for MOVNTDQ */
928 jnz Lsse2
929 /* Originally there was another case here for using the
930 * MOVNTQ instruction for processors that supported MMX but
931 * not SSE2. This turned out to be a loss especially on
932 * Athlons (where this instruction is apparently microcoded
933 * somewhat slowly). So for simplicity revert to REP STOSL
934 * for all non-SSE2 processors.
935 */
936 Lbase:
937 movl $(GNAME(fast_bzero_base)), GNAME(fast_bzero_pointer)
938 jmp Lrestore
939 Lsse2:
940 movl $(GNAME(fast_bzero_sse)), GNAME(fast_bzero_pointer)
941 jmp Lrestore
943 Lrestore:
944 pop %edx
945 pop %ecx
946 pop %ebx
947 pop %eax
948 jmp *GNAME(fast_bzero_pointer)
950 SIZE(GNAME(fast_bzero_detect))
952 \f
953 .text
954 .align align_16byte,0x90
955 .globl GNAME(fast_bzero_sse)
956 TYPE(GNAME(fast_bzero_sse))
958 GNAME(fast_bzero_sse):
959 /* A fast routine for zero-filling blocks of memory that are
960 * guaranteed to start and end at a 4096-byte aligned address.
961 */
962 push %esi /* Save temporary registers */
963 push %edi
964 mov 16(%esp), %esi /* Parameter: amount of bytes to fill */
965 mov 12(%esp), %edi /* Parameter: start address */
966 shr $6, %esi /* Amount of 64-byte blocks to copy */
967 jz Lend_sse /* If none, stop */
968 movups %xmm7, -16(%esp) /* Save XMM register */
969 xorps %xmm7, %xmm7 /* Zero the XMM register */
970 jmp Lloop_sse
971 .align align_16byte
972 Lloop_sse:
974 /* Copy the 16 zeroes from xmm7 to memory, 4 times. MOVNTDQ is the
975 * non-caching double-quadword moving variant, i.e. the memory areas
976 * we're touching are not fetched into the L1 cache, since we're just
977 * going to overwrite the memory soon anyway.
978 */
979 movntdq %xmm7, 0(%edi)
980 movntdq %xmm7, 16(%edi)
981 movntdq %xmm7, 32(%edi)
982 movntdq %xmm7, 48(%edi)
984 add $64, %edi /* Advance pointer */
985 dec %esi /* Decrement 64-byte block count */
986 jnz Lloop_sse
987 movups -16(%esp), %xmm7 /* Restore the XMM register */
988 sfence /* Ensure that weakly ordered writes are flushed. */
989 Lend_sse:
990 mov 12(%esp), %esi /* Parameter: start address */
991 prefetcht0 0(%esi) /* Prefetch the start of the block into cache,
992 * since it's likely to be used immediately. */
993 pop %edi /* Restore temp registers */
994 pop %esi
995 ret
996 SIZE(GNAME(fast_bzero_sse))
998 \f
999 .text
1000 .align align_16byte,0x90
1001 .globl GNAME(fast_bzero_base)
1002 TYPE(GNAME(fast_bzero_base))
1004 GNAME(fast_bzero_base):
1005 /* A fast routine for zero-filling blocks of memory that are
1006 * guaranteed to start and end at a 4096-byte aligned address.
1007 */
1008 push %eax /* Save temporary registers */
1009 push %ecx
1010 push %edi
1011 mov 20(%esp), %ecx /* Parameter: amount of bytes to fill */
1012 mov 16(%esp), %edi /* Parameter: start address */
1013 xor %eax, %eax /* Zero EAX */
1014 shr $2, %ecx /* Amount of 4-byte blocks to copy */
1015 jz Lend_base
1017 rep
1018 stosl /* Store EAX to *EDI, ECX times, incrementing
1019 * EDI by 4 after each store */
1021 Lend_base:
1022 pop %edi /* Restore temp registers */
1023 pop %ecx
1024 pop %eax
1025 ret
1026 SIZE(GNAME(fast_bzero_base))
1028 \f
1029 /* When LISP_FEATURE_C_STACK_IS_CONTROL_STACK, we cannot safely scrub
1030 * the control stack from C, largely due to not knowing where the
1031 * active stack frame ends. On such platforms, we reimplement the
1032 * core scrubbing logic in assembly, in this case here:
1033 */
1034 .text
1035 .align align_16byte,0x90
1036 .globl GNAME(arch_scrub_control_stack)
1037 TYPE(GNAME(arch_scrub_control_stack))
1038 GNAME(arch_scrub_control_stack):
1039 /* We are passed three parameters:
1040 * A (struct thread *) at [ESP+4],
1041 * the address of the guard page at [ESP+8], and
1042 * the address of the hard guard page at [ESP+12].
1043 * We may trash EAX, ECX, and EDX with impunity.
1044 * [ESP] is our return address, [ESP-4] is the first
1045 * stack slot to scrub. */
1047 /* We start by setting up our scrub pointer in EAX, our
1048 * guard page upper bound in ECX, and our hard guard
1049 * page upper bound in EDX. */
1050 lea -4(%esp), %eax
1051 mov GNAME(os_vm_page_size),%edx
1052 mov %edx, %ecx
1053 add 8(%esp), %ecx
1054 add 12(%esp), %edx
1056 /* We need to do a memory operation relative to the
1057 * thread pointer, so put it in %ecx and our guard
1058 * page upper bound in 4(%esp). */
1059 xchg 4(%esp), %ecx
1061 /* Now we begin our main scrub loop. */
1062 ascs_outer_loop:
1064 /* If we're about to scrub the hard guard page, exit. */
1065 cmp %edx, %eax
1066 jae ascs_check_guard_page
1067 cmp 12(%esp), %eax
1068 ja ascs_finished
1070 ascs_check_guard_page:
1071 /* If we're about to scrub the guard page, and the guard
1072 * page is protected, exit. */
1073 cmp 4(%esp), %eax
1074 jae ascs_clear_loop
1075 cmp 8(%esp), %eax
1076 jbe ascs_clear_loop
1077 cmpl $(NIL), THREAD_CONTROL_STACK_GUARD_PAGE_PROTECTED_OFFSET(%ecx)
1078 jne ascs_finished
1080 /* Clear memory backwards to the start of the (4KiB) page */
1081 ascs_clear_loop:
1082 movl $0, (%eax)
1083 test $0xfff, %eax
1084 lea -4(%eax), %eax
1085 jnz ascs_clear_loop
1087 /* If we're about to hit the hard guard page, exit. */
1088 cmp %edx, %eax
1089 jae ascs_finished
1091 /* If the next (previous?) 4KiB page contains a non-zero
1092 * word, continue scrubbing. */
1093 ascs_check_loop:
1094 testl $-1, (%eax)
1095 jnz ascs_outer_loop
1096 test $0xfff, %eax
1097 lea -4(%eax), %eax
1098 jnz ascs_check_loop
1100 ascs_finished:
1101 ret
1102 SIZE(GNAME(arch_scrub_control_stack))