src/compiler/x86-64/c-call.lisp

   1 ;;;; the VOPs and other necessary machine specific support
   2 ;;;; routines for call-out to C
   3
   4 ;;;; This software is part of the SBCL system. See the README file for
   5 ;;;; more information.
   6 ;;;;
   7 ;;;; This software is derived from the CMU CL system, which was
   8 ;;;; written at Carnegie Mellon University and released into the
   9 ;;;; public domain. The software is in the public domain and is
  10 ;;;; provided with absolutely no warranty. See the COPYING and CREDITS
  11 ;;;; files for more information.
  12
  13 (in-package "SB!VM")
  14
  15 ;; The MOVE-ARG vop is going to store args on the stack for
  16 ;; call-out. These tn's will be used for that. move-arg is normally
  17 ;; used for things going down the stack but C wants to have args
  18 ;; indexed in the positive direction.
  19
  20 (defstruct (arg-state (:copier nil))
  21   (register-args 0)
  22   (xmm-args 0)
  23   (stack-frame-size 0))
  24
  25 (defconstant max-int-args #.(length *c-call-register-arg-offsets*))
  26 (defconstant max-xmm-args #!+win32 4 #!-win32 8)
  27
  28 (defun int-arg (state prim-type reg-sc stack-sc)
  29   (let ((reg-args (max (arg-state-register-args state)
  30                        #!+win32 (arg-state-xmm-args state))))
  31     (cond ((< reg-args max-int-args)
  32            (setf (arg-state-register-args state) (1+ reg-args))
  33            (make-wired-tn* prim-type reg-sc
  34                              (nth reg-args *c-call-register-arg-offsets*)))
  35           (t
  36            (let ((frame-size (arg-state-stack-frame-size state)))
  37              (setf (arg-state-stack-frame-size state) (1+ frame-size))
  38              (make-wired-tn* prim-type stack-sc frame-size))))))
  39
  40 (define-alien-type-method (integer :arg-tn) (type state)
  41   (if (alien-integer-type-signed type)
  42       (int-arg state 'signed-byte-64 signed-reg-sc-number signed-stack-sc-number)
  43       (int-arg state 'unsigned-byte-64 unsigned-reg-sc-number unsigned-stack-sc-number)))
  44
  45 (define-alien-type-method (system-area-pointer :arg-tn) (type state)
  46   (declare (ignore type))
  47   (int-arg state 'system-area-pointer sap-reg-sc-number sap-stack-sc-number))
  48
  49 (defun float-arg (state prim-type reg-sc stack-sc)
  50   (let ((xmm-args (max (arg-state-xmm-args state)
  51                         #!+win32 (arg-state-register-args state))))
  52     (cond ((< xmm-args max-xmm-args)
  53            (setf (arg-state-xmm-args state) (1+ xmm-args))
  54            (make-wired-tn* prim-type reg-sc
  55                              (nth xmm-args *float-regs*)))
  56           (t
  57            (let ((frame-size (arg-state-stack-frame-size state)))
  58              (setf (arg-state-stack-frame-size state) (1+ frame-size))
  59              (make-wired-tn* prim-type stack-sc frame-size))))))
  60
  61 (define-alien-type-method (double-float :arg-tn) (type state)
  62   (declare (ignore type))
  63   (float-arg state 'double-float double-reg-sc-number double-stack-sc-number))
  64
  65 (define-alien-type-method (single-float :arg-tn) (type state)
  66   (declare (ignore type))
  67   (float-arg state 'single-float single-reg-sc-number single-stack-sc-number))
  68
  69 (defstruct (result-state (:copier nil))
  70   (num-results 0))
  71
  72 (defun result-reg-offset (slot)
  73   (ecase slot
  74     (0 eax-offset)
  75     (1 edx-offset)))
  76
  77 (define-alien-type-method (integer :result-tn) (type state)
  78   (let ((num-results (result-state-num-results state)))
  79     (setf (result-state-num-results state) (1+ num-results))
  80     (multiple-value-bind (ptype reg-sc)
  81         (if (alien-integer-type-signed type)
  82             (values 'signed-byte-64 signed-reg-sc-number)
  83             (values 'unsigned-byte-64 unsigned-reg-sc-number))
  84       (make-wired-tn* ptype reg-sc (result-reg-offset num-results)))))
  85
  86 (define-alien-type-method (integer :naturalize-gen) (type alien)
  87   (if (<= (alien-type-bits type) 32)
  88       (if (alien-integer-type-signed type)
  89           `(sign-extend ,alien ,(alien-type-bits type))
  90           `(logand ,alien ,(1- (ash 1 (alien-type-bits type)))))
  91       alien))
  92
  93 (define-alien-type-method (system-area-pointer :result-tn) (type state)
  94   (declare (ignore type))
  95   (let ((num-results (result-state-num-results state)))
  96     (setf (result-state-num-results state) (1+ num-results))
  97     (make-wired-tn* 'system-area-pointer sap-reg-sc-number
  98                       (result-reg-offset num-results))))
  99
 100 (define-alien-type-method (double-float :result-tn) (type state)
 101   (declare (ignore type))
 102   (let ((num-results (result-state-num-results state)))
 103     (setf (result-state-num-results state) (1+ num-results))
 104     (make-wired-tn* 'double-float double-reg-sc-number num-results)))
 105
 106 (define-alien-type-method (single-float :result-tn) (type state)
 107   (declare (ignore type))
 108   (let ((num-results (result-state-num-results state)))
 109     (setf (result-state-num-results state) (1+ num-results))
 110     (make-wired-tn* 'single-float single-reg-sc-number num-results)))
 111
 112 (define-alien-type-method (values :result-tn) (type state)
 113   (let ((values (alien-values-type-values type)))
 114     (when (> (length values) 2)
 115       (error "Too many result values from c-call."))
 116     (mapcar (lambda (type)
 117               (invoke-alien-type-method :result-tn type state))
 118             values)))
 119
 120 (defun make-call-out-tns (type)
 121   (let ((arg-state (make-arg-state)))
 122     (collect ((arg-tns))
 123       (dolist (arg-type (alien-fun-type-arg-types type))
 124         (arg-tns (invoke-alien-type-method :arg-tn arg-type arg-state)))
 125       (values (make-wired-tn* 'positive-fixnum any-reg-sc-number esp-offset)
 126               (* (arg-state-stack-frame-size arg-state) n-word-bytes)
 127               (arg-tns)
 128               (invoke-alien-type-method :result-tn
 129                                         (alien-fun-type-result-type type)
 130                                         (make-result-state))))))
 131
 132
 133 (deftransform %alien-funcall ((function type &rest args) * * :node node)
 134   (aver (sb!c::constant-lvar-p type))
 135   (let* ((type (sb!c::lvar-value type))
 136          (env (sb!c::node-lexenv node))
 137          (arg-types (alien-fun-type-arg-types type))
 138          (result-type (alien-fun-type-result-type type)))
 139     (aver (= (length arg-types) (length args)))
 140     (if (or (some #'(lambda (type)
 141                       (and (alien-integer-type-p type)
 142                            (> (sb!alien::alien-integer-type-bits type) 64)))
 143                   arg-types)
 144             (and (alien-integer-type-p result-type)
 145                  (> (sb!alien::alien-integer-type-bits result-type) 64)))
 146         (collect ((new-args) (lambda-vars) (new-arg-types))
 147           (dolist (type arg-types)
 148             (let ((arg (gensym)))
 149               (lambda-vars arg)
 150               (cond ((and (alien-integer-type-p type)
 151                           (> (sb!alien::alien-integer-type-bits type) 64))
 152                      ;; CLH: FIXME! This should really be
 153                      ;; #xffffffffffffffff. nyef says: "Passing
 154                      ;; 128-bit integers to ALIEN functions on x86-64
 155                      ;; believed to be broken."
 156                      (new-args `(logand ,arg #xffffffff))
 157                      (new-args `(ash ,arg -64))
 158                      (new-arg-types (parse-alien-type '(unsigned 64) env))
 159                      (if (alien-integer-type-signed type)
 160                          (new-arg-types (parse-alien-type '(signed 64) env))
 161                          (new-arg-types (parse-alien-type '(unsigned 64) env))))
 162                     (t
 163                      (new-args arg)
 164                      (new-arg-types type)))))
 165           (cond ((and (alien-integer-type-p result-type)
 166                       (> (sb!alien::alien-integer-type-bits result-type) 64))
 167                  (let ((new-result-type
 168                         (let ((sb!alien::*values-type-okay* t))
 169                           (parse-alien-type
 170                            (if (alien-integer-type-signed result-type)
 171                                '(values (unsigned 64) (signed 64))
 172                                '(values (unsigned 64) (unsigned 64)))
 173                            env))))
 174                    `(lambda (function type ,@(lambda-vars))
 175                       (declare (ignore type))
 176                       (multiple-value-bind (low high)
 177                           (%alien-funcall function
 178                                           ',(make-alien-fun-type
 179                                              :arg-types (new-arg-types)
 180                                              :result-type new-result-type)
 181                                           ,@(new-args))
 182                         (logior low (ash high 64))))))
 183                 (t
 184                  `(lambda (function type ,@(lambda-vars))
 185                     (declare (ignore type))
 186                     (%alien-funcall function
 187                                     ',(make-alien-fun-type
 188                                        :arg-types (new-arg-types)
 189                                        :result-type result-type)
 190                                     ,@(new-args))))))
 191         (sb!c::give-up-ir1-transform))))
 192
 193 ;;; The ABI is vague about how signed sub-word integer return values
 194 ;;; are handled, but since gcc versions >=4.3 no longer do sign
 195 ;;; extension in the callee, we need to do it in the caller.  FIXME:
 196 ;;; If the value to be extended is known to already be of the target
 197 ;;; type at compile time, we can (and should) elide the extension.
 198 (defknown sign-extend ((signed-byte 64) t) fixnum
 199     (foldable flushable movable))
 200
 201 (define-vop (sign-extend)
 202   (:translate sign-extend)
 203   (:policy :fast-safe)
 204   (:args (val :scs (signed-reg)))
 205   (:arg-types signed-num (:constant fixnum))
 206   (:info size)
 207   (:results (res :scs (signed-reg)))
 208   (:result-types fixnum)
 209   (:generator 1
 210    (inst movsxd res
 211          (make-random-tn :kind :normal
 212                          :sc (sc-or-lose (ecase size
 213                                            (8 'byte-reg)
 214                                            (16 'word-reg)
 215                                            (32 'dword-reg)))
 216                          :offset (tn-offset val)))))
 217
 218 #-sb-xc-host
 219 (defun sign-extend (x size)
 220   (declare (type (signed-byte 64) x))
 221   (ecase size
 222     (8 (sign-extend x size))
 223     (16 (sign-extend x size))
 224     (32 (sign-extend x size))))
 225
 226 #+sb-xc-host
 227 (defun sign-extend (x size)
 228   (if (logbitp (1- size) x)
 229       (dpb x (byte size 0) -1)
 230       x))
 231
 232 (define-vop (foreign-symbol-sap)
 233   (:translate foreign-symbol-sap)
 234   (:policy :fast-safe)
 235   (:args)
 236   (:arg-types (:constant simple-string))
 237   (:info foreign-symbol)
 238   (:results (res :scs (sap-reg)))
 239   (:result-types system-area-pointer)
 240   (:generator 2
 241    (inst mov res (make-fixup foreign-symbol :foreign))))
 242
 243 #!+linkage-table
 244 (define-vop (foreign-symbol-dataref-sap)
 245   (:translate foreign-symbol-dataref-sap)
 246   (:policy :fast-safe)
 247   (:args)
 248   (:arg-types (:constant simple-string))
 249   (:info foreign-symbol)
 250   (:results (res :scs (sap-reg)))
 251   (:result-types system-area-pointer)
 252   (:generator 2
 253    (inst mov res (make-fixup foreign-symbol :foreign-dataref))))
 254
 255 (define-vop (call-out)
 256   (:args (function :scs (sap-reg)
 257                    :target rbx)
 258          (args :more t))
 259   (:results (results :more t))
 260   ;; RBX is used to first load the address, allowing the debugger to
 261   ;; determine which alien was accessed in case it's undefined.
 262   (:temporary (:sc sap-reg :offset rbx-offset :from (:argument 0)) rbx)
 263   (:temporary (:sc unsigned-reg :offset rax-offset :to :result) rax)
 264   ;; For safepoint builds: Force values of non-volatiles to the stack.
 265   ;; These are the callee-saved registers in the native ABI, but
 266   ;; safepoint-based GC needs to see all Lisp values on the stack.  Note
 267   ;; that R12-R15 are non-volatile registers, but there is no need to
 268   ;; spill R12 because it is our thread-base-tn.  RDI and RSI are
 269   ;; non-volatile on Windows, but argument passing registers on other
 270   ;; platforms.
 271   #!+sb-safepoint (:temporary (:sc unsigned-reg :offset r13-offset) r13)
 272   #!+sb-safepoint (:temporary (:sc unsigned-reg :offset r14-offset) r14)
 273   #!+sb-safepoint (:temporary (:sc unsigned-reg :offset r15-offset) r15)
 274   #!+(and sb-safepoint win32) (:temporary
 275                                (:sc unsigned-reg :offset rdi-offset) rdi)
 276   #!+(and sb-safepoint win32) (:temporary
 277                                (:sc unsigned-reg :offset rsi-offset) rsi)
 278   (:ignore results
 279            #!+(and sb-safepoint win32) rdi
 280            #!+(and sb-safepoint win32) rsi
 281            #!+sb-safepoint r15
 282            #!+sb-safepoint r13)
 283   (:vop-var vop)
 284   (:save-p t)
 285   (:generator 0
 286     #!+sb-safepoint
 287     (progn
 288       ;; Current PC - don't rely on function to keep it in a form that
 289       ;; GC understands
 290       (let ((label (gen-label)))
 291         (inst lea r14 (make-fixup nil :code-object label))
 292         (emit-label label)))
 293     (when sb!c::*msan-compatible-stack-unpoison*
 294       (inst mov rax (static-symbol-value-ea 'msan-param-tls))
 295       ;; Unpoison parameters
 296       (do ((n 0 (+ n n-word-bytes))
 297            (arg args (tn-ref-across arg)))
 298           ((null arg))
 299         ;; KLUDGE: assume all parameters are 8 bytes or less
 300         (inst fs)
 301         (inst mov (make-ea :qword :base rax :disp n) 0)))
 302     #!-win32
 303     ;; ABI: AL contains amount of arguments passed in XMM registers
 304     ;; for vararg calls.
 305     (move-immediate rax
 306                     (loop for tn-ref = args then (tn-ref-across tn-ref)
 307                           while tn-ref
 308                           count (eq (sb-name (sc-sb (tn-sc (tn-ref-tn tn-ref))))
 309                                     'float-registers)))
 310     #!+win32 (inst sub rsp-tn #x20) ;MS_ABI: shadow zone
 311     #!+sb-safepoint
 312     (progn                 ;Store SP and PC in thread struct
 313       (storew rsp-tn thread-base-tn thread-saved-csp-offset)
 314       (storew r14 thread-base-tn thread-pc-around-foreign-call-slot))
 315     (move rbx function)
 316     (inst call rbx)
 317     #!+win32 (inst add rsp-tn #x20) ;MS_ABI: remove shadow space
 318     #!+sb-safepoint
 319     (progn
 320       ;; Zeroing out
 321       (inst xor r14 r14)
 322       ;; Zero PC storage place. NB. CSP-then-PC: same sequence on
 323       ;; entry/exit, is actually corrent.
 324       (storew r14 thread-base-tn thread-saved-csp-offset)
 325       (storew r14 thread-base-tn thread-pc-around-foreign-call-slot))
 326     ;; To give the debugger a clue. XX not really internal-error?
 327     (note-this-location vop :internal-error)))
 328
 329 (define-vop (alloc-number-stack-space)
 330   (:info amount)
 331   (:results (result :scs (sap-reg any-reg)))
 332   (:result-types system-area-pointer)
 333   (:generator 0
 334     (aver (location= result rsp-tn))
 335     (unless (zerop amount)
 336       (let ((delta (logandc2 (+ amount 7) 7)))
 337         (inst sub rsp-tn delta)))
 338     ;; C stack must be 16 byte aligned
 339     (inst and rsp-tn -16)
 340     (move result rsp-tn)))
 341
 342 (macrolet ((alien-stack-ptr ()
 343              #!+sb-thread '(symbol-known-tls-cell '*alien-stack-pointer*)
 344              #!-sb-thread '(static-symbol-value-ea '*alien-stack-pointer*)))
 345   (define-vop (alloc-alien-stack-space)
 346     (:info amount)
 347     (:results (result :scs (sap-reg any-reg)))
 348     (:result-types system-area-pointer)
 349     (:generator 0
 350       (aver (not (location= result rsp-tn)))
 351       (unless (zerop amount)
 352         (let ((delta (logandc2 (+ amount 7) 7)))
 353           (inst sub (alien-stack-ptr) delta)))
 354       (inst mov result (alien-stack-ptr)))))
 355
 356 ;;; not strictly part of the c-call convention, but needed for the
 357 ;;; WITH-PINNED-OBJECTS macro used for "locking down" lisp objects so
 358 ;;; that GC won't move them while foreign functions go to work.
 359 (define-vop (touch-object)
 360   (:translate touch-object)
 361   (:args (object))
 362   (:ignore object)
 363   (:policy :fast-safe)
 364   (:arg-types t)
 365   (:generator 0))
 366
 367 ;;; Callbacks
 368
 369 #-sb-xc-host
 370 (defun alien-callback-accessor-form (type sp offset)
 371   `(deref (sap-alien (sap+ ,sp ,offset) (* ,type))))
 372
 373 #-sb-xc-host
 374 (defun alien-callback-assembler-wrapper (index result-type argument-types)
 375   (labels ((make-tn-maker (sc-name)
 376              (lambda (offset)
 377                (make-random-tn :kind :normal
 378                                :sc (sc-or-lose sc-name)
 379                                :offset offset))))
 380     (let* ((segment (make-segment))
 381            (rax rax-tn)
 382            #!+(or win32 (not sb-thread)) (rcx rcx-tn)
 383            #!-(and win32 sb-thread) (rdi rdi-tn)
 384            #!-(and win32 sb-thread) (rsi rsi-tn)
 385            (rdx rdx-tn)
 386            (rbp rbp-tn)
 387            (rsp rsp-tn)
 388            #!+(and win32 sb-thread) (r8 r8-tn)
 389            (xmm0 float0-tn)
 390            ([rsp] (make-ea :qword :base rsp :disp 0))
 391            ;; How many arguments have been copied
 392            (arg-count 0)
 393            ;; How many arguments have been copied from the stack
 394            (stack-argument-count #!-win32 0 #!+win32 4)
 395            (gprs (mapcar (make-tn-maker 'any-reg) *c-call-register-arg-offsets*))
 396            (fprs (mapcar (make-tn-maker 'double-reg)
 397                          ;; Only 8 first XMM registers are used for
 398                          ;; passing arguments
 399                          (subseq *float-regs* 0 #!-win32 8 #!+win32 4))))
 400       (assemble (segment)
 401         ;; Make room on the stack for arguments.
 402         (when argument-types
 403           (inst sub rsp (* n-word-bytes (length argument-types))))
 404         ;; Copy arguments from registers to stack
 405         (dolist (type argument-types)
 406           (let ((integerp (not (alien-float-type-p type)))
 407                 ;; A TN pointing to the stack location where the
 408                 ;; current argument should be stored for the purposes
 409                 ;; of ENTER-ALIEN-CALLBACK.
 410                 (target-tn (make-ea :qword :base rsp
 411                                    :disp (* arg-count
 412                                             n-word-bytes)))
 413                 ;; A TN pointing to the stack location that contains
 414                 ;; the next argument passed on the stack.
 415                 (stack-arg-tn (make-ea :qword :base rsp
 416                                        :disp (* (+ 1
 417                                                    (length argument-types)
 418                                                    stack-argument-count)
 419                                                 n-word-bytes))))
 420             (incf arg-count)
 421             (cond (integerp
 422                    (let ((gpr (pop gprs)))
 423                      #!+win32 (pop fprs)
 424                      ;; Argument not in register, copy it from the old
 425                      ;; stack location to a temporary register.
 426                      (unless gpr
 427                        (incf stack-argument-count)
 428                        (setf gpr temp-reg-tn)
 429                        (inst mov gpr stack-arg-tn))
 430                      ;; Copy from either argument register or temporary
 431                      ;; register to target.
 432                      (inst mov target-tn gpr)))
 433                   ((or (alien-single-float-type-p type)
 434                        (alien-double-float-type-p type))
 435                    (let ((fpr (pop fprs)))
 436                      #!+win32 (pop gprs)
 437                      (cond (fpr
 438                             ;; Copy from float register to target location.
 439                             (inst movq target-tn fpr))
 440                            (t
 441                             ;; Not in float register. Copy from stack to
 442                             ;; temporary (general purpose) register, and
 443                             ;; from there to the target location.
 444                             (incf stack-argument-count)
 445                             (inst mov temp-reg-tn stack-arg-tn)
 446                             (inst mov target-tn temp-reg-tn)))))
 447                   (t
 448                    (bug "Unknown alien floating point type: ~S" type)))))
 449
 450         #!-sb-thread
 451         (progn
 452           ;; arg0 to FUNCALL3 (function)
 453           (inst mov rdi (make-ea :qword :disp (static-fdefn-fun-addr 'enter-alien-callback)))
 454           ;; arg0 to ENTER-ALIEN-CALLBACK (trampoline index)
 455           (inst mov rsi (fixnumize index))
 456           ;; arg1 to ENTER-ALIEN-CALLBACK (pointer to argument vector)
 457           (inst mov rdx rsp)
 458           ;; add room on stack for return value
 459           (inst sub rsp (if (evenp arg-count)
 460                             (* n-word-bytes 2)
 461                             n-word-bytes))
 462           ;; arg2 to ENTER-ALIEN-CALLBACK (pointer to return value)
 463           (inst mov rcx rsp)
 464
 465           ;; Make new frame
 466           (inst push rbp)
 467           (inst mov  rbp rsp)
 468
 469           ;; Call
 470           (inst mov  rax (foreign-symbol-address "funcall3"))
 471           (inst call rax)
 472
 473           ;; Back! Restore frame
 474           (inst mov rsp rbp)
 475           (inst pop rbp))
 476
 477         #!+sb-thread
 478         (progn
 479           ;; arg0 to ENTER-ALIEN-CALLBACK (trampoline index)
 480           (inst mov #!-win32 rdi #!+win32 rcx (fixnumize index))
 481           ;; arg1 to ENTER-ALIEN-CALLBACK (pointer to argument vector)
 482           (inst mov #!-win32 rsi #!+win32 rdx rsp)
 483           ;; add room on stack for return value
 484           (inst sub rsp (if (evenp arg-count)
 485                             (* n-word-bytes 2)
 486                             n-word-bytes))
 487           ;; arg2 to ENTER-ALIEN-CALLBACK (pointer to return value)
 488           (inst mov #!-win32 rdx #!+win32 r8 rsp)
 489           ;; Make new frame
 490           (inst push rbp)
 491           (inst mov  rbp rsp)
 492           #!+win32 (inst sub rsp #x20)
 493           #!+win32 (inst and rsp #x-20)
 494           ;; Call
 495           (inst mov rax (foreign-symbol-address "callback_wrapper_trampoline"))
 496           (inst call rax)
 497           ;; Back! Restore frame
 498           (inst mov rsp rbp)
 499           (inst pop rbp))
 500
 501         ;; Result now on top of stack, put it in the right register
 502         (cond
 503           ((or (alien-integer-type-p result-type)
 504                (alien-pointer-type-p result-type)
 505                (alien-type-= #.(parse-alien-type 'system-area-pointer nil)
 506                              result-type))
 507            (inst mov rax [rsp]))
 508           ((or (alien-single-float-type-p result-type)
 509                (alien-double-float-type-p result-type))
 510            (inst movq xmm0 [rsp]))
 511           ((alien-void-type-p result-type))
 512           (t
 513            (error "Unrecognized alien type: ~A" result-type)))
 514
 515         ;; Pop the arguments and the return value from the stack to get
 516         ;; the return address at top of stack.
 517
 518         (inst add rsp (* (+ arg-count
 519                             ;; Plus the return value and make sure it's aligned
 520                             (if (evenp arg-count)
 521                                 2
 522                                 1))
 523                          n-word-bytes))
 524         ;; Return
 525         (inst ret))
 526       (finalize-segment segment)
 527       ;; Now that the segment is done, convert it to a static
 528       ;; vector we can point foreign code to.
 529       (let ((buffer (sb!assem::segment-buffer segment)))
 530         (make-static-vector (length buffer)
 531                             :element-type '(unsigned-byte 8)
 532                             :initial-contents buffer)))))