?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
x86, extable: Switch to relative exception table entries
[linux-2.6.git] / arch / x86 / include / asm / uaccess.h
blob851fe0dc13bc18c33c79b8d54deb06e899ae63ad
1 #ifndef _ASM_X86_UACCESS_H
2 #define _ASM_X86_UACCESS_H
3 /*
4 * User space memory access functions
5 */
6 #include <linux/errno.h>
7 #include <linux/compiler.h>
8 #include <linux/thread_info.h>
9 #include <linux/string.h>
10 #include <asm/asm.h>
11 #include <asm/page.h>
12
13 #define VERIFY_READ 0
14 #define VERIFY_WRITE 1
15
16 /*
17 * The fs value determines whether argument validity checking should be
18 * performed or not. If get_fs() == USER_DS, checking is performed, with
19 * get_fs() == KERNEL_DS, checking is bypassed.
20 *
21 * For historical reasons, these macros are grossly misnamed.
22 */
23
24 #define MAKE_MM_SEG(s) ((mm_segment_t) { (s) })
25
26 #define KERNEL_DS MAKE_MM_SEG(-1UL)
27 #define USER_DS MAKE_MM_SEG(TASK_SIZE_MAX)
28
29 #define get_ds() (KERNEL_DS)
30 #define get_fs() (current_thread_info()->addr_limit)
31 #define set_fs(x) (current_thread_info()->addr_limit = (x))
32
33 #define segment_eq(a, b) ((a).seg == (b).seg)
34
35 #define __addr_ok(addr) \
36 ((unsigned long __force)(addr) < \
37 (current_thread_info()->addr_limit.seg))
38
39 /*
40 * Test whether a block of memory is a valid user space address.
41 * Returns 0 if the range is valid, nonzero otherwise.
42 *
43 * This is equivalent to the following test:
44 * (u33)addr + (u33)size > (u33)current->addr_limit.seg (u65 for x86_64)
45 *
46 * This needs 33-bit (65-bit for x86_64) arithmetic. We have a carry...
47 */
48
49 #define __range_not_ok(addr, size) \
50 ({ \
51 unsigned long flag, roksum; \
52 __chk_user_ptr(addr); \
53 asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" \
54 : "=&r" (flag), "=r" (roksum) \
55 : "1" (addr), "g" ((long)(size)), \
56 "rm" (current_thread_info()->addr_limit.seg)); \
57 flag; \
58 })
59
60 /**
61 * access_ok: - Checks if a user space pointer is valid
62 * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE. Note that
63 * %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe
64 * to write to a block, it is always safe to read from it.
65 * @addr: User space pointer to start of block to check
66 * @size: Size of block to check
67 *
68 * Context: User context only. This function may sleep.
69 *
70 * Checks if a pointer to a block of memory in user space is valid.
71 *
72 * Returns true (nonzero) if the memory block may be valid, false (zero)
73 * if it is definitely invalid.
74 *
75 * Note that, depending on architecture, this function probably just
76 * checks that the pointer is in the user space range - after calling
77 * this function, memory access functions may still return -EFAULT.
78 */
79 #define access_ok(type, addr, size) (likely(__range_not_ok(addr, size) == 0))
80
81 /*
82 * The exception table consists of pairs of addresses relative to the
83 * exception table enty itself: the first is the address of an
84 * instruction that is allowed to fault, and the second is the address
85 * at which the program should continue. No registers are modified,
86 * so it is entirely up to the continuation code to figure out what to
87 * do.
88 *
89 * All the routines below use bits of fixup code that are out of line
90 * with the main instruction path. This means when everything is well,
91 * we don't even have to jump over them. Further, they do not intrude
92 * on our cache or tlb entries.
93 */
94
95 struct exception_table_entry {
96 int insn, fixup;
97 };
98 /* This is not the generic standard exception_table_entry format */
99 #define ARCH_HAS_SORT_EXTABLE
100 #define ARCH_HAS_SEARCH_EXTABLE
101
102 extern int fixup_exception(struct pt_regs *regs);
103 extern int early_fixup_exception(unsigned long *ip);
104
105 /*
106 * These are the main single-value transfer routines. They automatically
107 * use the right size if we just have the right pointer type.
108 *
109 * This gets kind of ugly. We want to return _two_ values in "get_user()"
110 * and yet we don't want to do any pointers, because that is too much
111 * of a performance impact. Thus we have a few rather ugly macros here,
112 * and hide all the ugliness from the user.
113 *
114 * The "__xxx" versions of the user access functions are versions that
115 * do not verify the address space, that must have been done previously
116 * with a separate "access_ok()" call (this is used when we do multiple
117 * accesses to the same area of user memory).
118 */
119
120 extern int __get_user_1(void);
121 extern int __get_user_2(void);
122 extern int __get_user_4(void);
123 extern int __get_user_8(void);
124 extern int __get_user_bad(void);
125
126 #define __get_user_x(size, ret, x, ptr) \
127 asm volatile("call __get_user_" #size \
128 : "=a" (ret), "=d" (x) \
129 : "0" (ptr)) \
130
131 /* Careful: we have to cast the result to the type of the pointer
132 * for sign reasons */
133
134 /**
135 * get_user: - Get a simple variable from user space.
136 * @x: Variable to store result.
137 * @ptr: Source address, in user space.
138 *
139 * Context: User context only. This function may sleep.
140 *
141 * This macro copies a single simple variable from user space to kernel
142 * space. It supports simple types like char and int, but not larger
143 * data types like structures or arrays.
144 *
145 * @ptr must have pointer-to-simple-variable type, and the result of
146 * dereferencing @ptr must be assignable to @x without a cast.
147 *
148 * Returns zero on success, or -EFAULT on error.
149 * On error, the variable @x is set to zero.
150 */
151 #ifdef CONFIG_X86_32
152 #define __get_user_8(__ret_gu, __val_gu, ptr) \
153 __get_user_x(X, __ret_gu, __val_gu, ptr)
154 #else
155 #define __get_user_8(__ret_gu, __val_gu, ptr) \
156 __get_user_x(8, __ret_gu, __val_gu, ptr)
157 #endif
158
159 #define get_user(x, ptr) \
160 ({ \
161 int __ret_gu; \
162 unsigned long __val_gu; \
163 __chk_user_ptr(ptr); \
164 might_fault(); \
165 switch (sizeof(*(ptr))) { \
166 case 1: \
167 __get_user_x(1, __ret_gu, __val_gu, ptr); \
168 break; \
169 case 2: \
170 __get_user_x(2, __ret_gu, __val_gu, ptr); \
171 break; \
172 case 4: \
173 __get_user_x(4, __ret_gu, __val_gu, ptr); \
174 break; \
175 case 8: \
176 __get_user_8(__ret_gu, __val_gu, ptr); \
177 break; \
178 default: \
179 __get_user_x(X, __ret_gu, __val_gu, ptr); \
180 break; \
181 } \
182 (x) = (__typeof__(*(ptr)))__val_gu; \
183 __ret_gu; \
184 })
185
186 #define __put_user_x(size, x, ptr, __ret_pu) \
187 asm volatile("call __put_user_" #size : "=a" (__ret_pu) \
188 : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
189
190
191
192 #ifdef CONFIG_X86_32
193 #define __put_user_asm_u64(x, addr, err, errret) \
194 asm volatile("1: movl %%eax,0(%2)\n" \
195 "2: movl %%edx,4(%2)\n" \
196 "3:\n" \
197 ".section .fixup,\"ax\"\n" \
198 "4: movl %3,%0\n" \
199 " jmp 3b\n" \
200 ".previous\n" \
201 _ASM_EXTABLE(1b, 4b) \
202 _ASM_EXTABLE(2b, 4b) \
203 : "=r" (err) \
204 : "A" (x), "r" (addr), "i" (errret), "0" (err))
205
206 #define __put_user_asm_ex_u64(x, addr) \
207 asm volatile("1: movl %%eax,0(%1)\n" \
208 "2: movl %%edx,4(%1)\n" \
209 "3:\n" \
210 _ASM_EXTABLE_EX(1b, 2b) \
211 _ASM_EXTABLE_EX(2b, 3b) \
212 : : "A" (x), "r" (addr))
213
214 #define __put_user_x8(x, ptr, __ret_pu) \
215 asm volatile("call __put_user_8" : "=a" (__ret_pu) \
216 : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
217 #else
218 #define __put_user_asm_u64(x, ptr, retval, errret) \
219 __put_user_asm(x, ptr, retval, "q", "", "er", errret)
220 #define __put_user_asm_ex_u64(x, addr) \
221 __put_user_asm_ex(x, addr, "q", "", "er")
222 #define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu)
223 #endif
224
225 extern void __put_user_bad(void);
226
227 /*
228 * Strange magic calling convention: pointer in %ecx,
229 * value in %eax(:%edx), return value in %eax. clobbers %rbx
230 */
231 extern void __put_user_1(void);
232 extern void __put_user_2(void);
233 extern void __put_user_4(void);
234 extern void __put_user_8(void);
235
236 #ifdef CONFIG_X86_WP_WORKS_OK
237
238 /**
239 * put_user: - Write a simple value into user space.
240 * @x: Value to copy to user space.
241 * @ptr: Destination address, in user space.
242 *
243 * Context: User context only. This function may sleep.
244 *
245 * This macro copies a single simple value from kernel space to user
246 * space. It supports simple types like char and int, but not larger
247 * data types like structures or arrays.
248 *
249 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
250 * to the result of dereferencing @ptr.
251 *
252 * Returns zero on success, or -EFAULT on error.
253 */
254 #define put_user(x, ptr) \
255 ({ \
256 int __ret_pu; \
257 __typeof__(*(ptr)) __pu_val; \
258 __chk_user_ptr(ptr); \
259 might_fault(); \
260 __pu_val = x; \
261 switch (sizeof(*(ptr))) { \
262 case 1: \
263 __put_user_x(1, __pu_val, ptr, __ret_pu); \
264 break; \
265 case 2: \
266 __put_user_x(2, __pu_val, ptr, __ret_pu); \
267 break; \
268 case 4: \
269 __put_user_x(4, __pu_val, ptr, __ret_pu); \
270 break; \
271 case 8: \
272 __put_user_x8(__pu_val, ptr, __ret_pu); \
273 break; \
274 default: \
275 __put_user_x(X, __pu_val, ptr, __ret_pu); \
276 break; \
277 } \
278 __ret_pu; \
279 })
280
281 #define __put_user_size(x, ptr, size, retval, errret) \
282 do { \
283 retval = 0; \
284 __chk_user_ptr(ptr); \
285 switch (size) { \
286 case 1: \
287 __put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \
288 break; \
289 case 2: \
290 __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \
291 break; \
292 case 4: \
293 __put_user_asm(x, ptr, retval, "l", "k", "ir", errret); \
294 break; \
295 case 8: \
296 __put_user_asm_u64((__typeof__(*ptr))(x), ptr, retval, \
297 errret); \
298 break; \
299 default: \
300 __put_user_bad(); \
301 } \
302 } while (0)
303
304 #define __put_user_size_ex(x, ptr, size) \
305 do { \
306 __chk_user_ptr(ptr); \
307 switch (size) { \
308 case 1: \
309 __put_user_asm_ex(x, ptr, "b", "b", "iq"); \
310 break; \
311 case 2: \
312 __put_user_asm_ex(x, ptr, "w", "w", "ir"); \
313 break; \
314 case 4: \
315 __put_user_asm_ex(x, ptr, "l", "k", "ir"); \
316 break; \
317 case 8: \
318 __put_user_asm_ex_u64((__typeof__(*ptr))(x), ptr); \
319 break; \
320 default: \
321 __put_user_bad(); \
322 } \
323 } while (0)
324
325 #else
326
327 #define __put_user_size(x, ptr, size, retval, errret) \
328 do { \
329 __typeof__(*(ptr))__pus_tmp = x; \
330 retval = 0; \
331 \
332 if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, size) != 0)) \
333 retval = errret; \
334 } while (0)
335
336 #define put_user(x, ptr) \
337 ({ \
338 int __ret_pu; \
339 __typeof__(*(ptr))__pus_tmp = x; \
340 __ret_pu = 0; \
341 if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, \
342 sizeof(*(ptr))) != 0)) \
343 __ret_pu = -EFAULT; \
344 __ret_pu; \
345 })
346 #endif
347
348 #ifdef CONFIG_X86_32
349 #define __get_user_asm_u64(x, ptr, retval, errret) (x) = __get_user_bad()
350 #define __get_user_asm_ex_u64(x, ptr) (x) = __get_user_bad()
351 #else
352 #define __get_user_asm_u64(x, ptr, retval, errret) \
353 __get_user_asm(x, ptr, retval, "q", "", "=r", errret)
354 #define __get_user_asm_ex_u64(x, ptr) \
355 __get_user_asm_ex(x, ptr, "q", "", "=r")
356 #endif
357
358 #define __get_user_size(x, ptr, size, retval, errret) \
359 do { \
360 retval = 0; \
361 __chk_user_ptr(ptr); \
362 switch (size) { \
363 case 1: \
364 __get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \
365 break; \
366 case 2: \
367 __get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \
368 break; \
369 case 4: \
370 __get_user_asm(x, ptr, retval, "l", "k", "=r", errret); \
371 break; \
372 case 8: \
373 __get_user_asm_u64(x, ptr, retval, errret); \
374 break; \
375 default: \
376 (x) = __get_user_bad(); \
377 } \
378 } while (0)
379
380 #define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \
381 asm volatile("1: mov"itype" %2,%"rtype"1\n" \
382 "2:\n" \
383 ".section .fixup,\"ax\"\n" \
384 "3: mov %3,%0\n" \
385 " xor"itype" %"rtype"1,%"rtype"1\n" \
386 " jmp 2b\n" \
387 ".previous\n" \
388 _ASM_EXTABLE(1b, 3b) \
389 : "=r" (err), ltype(x) \
390 : "m" (__m(addr)), "i" (errret), "0" (err))
391
392 #define __get_user_size_ex(x, ptr, size) \
393 do { \
394 __chk_user_ptr(ptr); \
395 switch (size) { \
396 case 1: \
397 __get_user_asm_ex(x, ptr, "b", "b", "=q"); \
398 break; \
399 case 2: \
400 __get_user_asm_ex(x, ptr, "w", "w", "=r"); \
401 break; \
402 case 4: \
403 __get_user_asm_ex(x, ptr, "l", "k", "=r"); \
404 break; \
405 case 8: \
406 __get_user_asm_ex_u64(x, ptr); \
407 break; \
408 default: \
409 (x) = __get_user_bad(); \
410 } \
411 } while (0)
412
413 #define __get_user_asm_ex(x, addr, itype, rtype, ltype) \
414 asm volatile("1: mov"itype" %1,%"rtype"0\n" \
415 "2:\n" \
416 _ASM_EXTABLE_EX(1b, 2b) \
417 : ltype(x) : "m" (__m(addr)))
418
419 #define __put_user_nocheck(x, ptr, size) \
420 ({ \
421 int __pu_err; \
422 __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \
423 __pu_err; \
424 })
425
426 #define __get_user_nocheck(x, ptr, size) \
427 ({ \
428 int __gu_err; \
429 unsigned long __gu_val; \
430 __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \
431 (x) = (__force __typeof__(*(ptr)))__gu_val; \
432 __gu_err; \
433 })
434
435 /* FIXME: this hack is definitely wrong -AK */
436 struct __large_struct { unsigned long buf[100]; };
437 #define __m(x) (*(struct __large_struct __user *)(x))
438
439 /*
440 * Tell gcc we read from memory instead of writing: this is because
441 * we do not write to any memory gcc knows about, so there are no
442 * aliasing issues.
443 */
444 #define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \
445 asm volatile("1: mov"itype" %"rtype"1,%2\n" \
446 "2:\n" \
447 ".section .fixup,\"ax\"\n" \
448 "3: mov %3,%0\n" \
449 " jmp 2b\n" \
450 ".previous\n" \
451 _ASM_EXTABLE(1b, 3b) \
452 : "=r"(err) \
453 : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err))
454
455 #define __put_user_asm_ex(x, addr, itype, rtype, ltype) \
456 asm volatile("1: mov"itype" %"rtype"0,%1\n" \
457 "2:\n" \
458 _ASM_EXTABLE_EX(1b, 2b) \
459 : : ltype(x), "m" (__m(addr)))
460
461 /*
462 * uaccess_try and catch
463 */
464 #define uaccess_try do { \
465 int prev_err = current_thread_info()->uaccess_err; \
466 current_thread_info()->uaccess_err = 0; \
467 barrier();
468
469 #define uaccess_catch(err) \
470 (err) |= (current_thread_info()->uaccess_err ? -EFAULT : 0); \
471 current_thread_info()->uaccess_err = prev_err; \
472 } while (0)
473
474 /**
475 * __get_user: - Get a simple variable from user space, with less checking.
476 * @x: Variable to store result.
477 * @ptr: Source address, in user space.
478 *
479 * Context: User context only. This function may sleep.
480 *
481 * This macro copies a single simple variable from user space to kernel
482 * space. It supports simple types like char and int, but not larger
483 * data types like structures or arrays.
484 *
485 * @ptr must have pointer-to-simple-variable type, and the result of
486 * dereferencing @ptr must be assignable to @x without a cast.
487 *
488 * Caller must check the pointer with access_ok() before calling this
489 * function.
490 *
491 * Returns zero on success, or -EFAULT on error.
492 * On error, the variable @x is set to zero.
493 */
494
495 #define __get_user(x, ptr) \
496 __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
497
498 /**
499 * __put_user: - Write a simple value into user space, with less checking.
500 * @x: Value to copy to user space.
501 * @ptr: Destination address, in user space.
502 *
503 * Context: User context only. This function may sleep.
504 *
505 * This macro copies a single simple value from kernel space to user
506 * space. It supports simple types like char and int, but not larger
507 * data types like structures or arrays.
508 *
509 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
510 * to the result of dereferencing @ptr.
511 *
512 * Caller must check the pointer with access_ok() before calling this
513 * function.
514 *
515 * Returns zero on success, or -EFAULT on error.
516 */
517
518 #define __put_user(x, ptr) \
519 __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
520
521 #define __get_user_unaligned __get_user
522 #define __put_user_unaligned __put_user
523
524 /*
525 * {get|put}_user_try and catch
526 *
527 * get_user_try {
528 * get_user_ex(...);
529 * } get_user_catch(err)
530 */
531 #define get_user_try uaccess_try
532 #define get_user_catch(err) uaccess_catch(err)
533
534 #define get_user_ex(x, ptr) do { \
535 unsigned long __gue_val; \
536 __get_user_size_ex((__gue_val), (ptr), (sizeof(*(ptr)))); \
537 (x) = (__force __typeof__(*(ptr)))__gue_val; \
538 } while (0)
539
540 #ifdef CONFIG_X86_WP_WORKS_OK
541
542 #define put_user_try uaccess_try
543 #define put_user_catch(err) uaccess_catch(err)
544
545 #define put_user_ex(x, ptr) \
546 __put_user_size_ex((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
547
548 #else /* !CONFIG_X86_WP_WORKS_OK */
549
550 #define put_user_try do { \
551 int __uaccess_err = 0;
552
553 #define put_user_catch(err) \
554 (err) |= __uaccess_err; \
555 } while (0)
556
557 #define put_user_ex(x, ptr) do { \
558 __uaccess_err |= __put_user(x, ptr); \
559 } while (0)
560
561 #endif /* CONFIG_X86_WP_WORKS_OK */
562
563 extern unsigned long
564 copy_from_user_nmi(void *to, const void __user *from, unsigned long n);
565 extern __must_check long
566 strncpy_from_user(char *dst, const char __user *src, long count);
567
568 /*
569 * movsl can be slow when source and dest are not both 8-byte aligned
570 */
571 #ifdef CONFIG_X86_INTEL_USERCOPY
572 extern struct movsl_mask {
573 int mask;
574 } ____cacheline_aligned_in_smp movsl_mask;
575 #endif
576
577 #define ARCH_HAS_NOCACHE_UACCESS 1
578
579 #ifdef CONFIG_X86_32
580 # include "uaccess_32.h"
581 #else
582 # include "uaccess_64.h"
583 #endif
584
585 #endif /* _ASM_X86_UACCESS_H */
586
Linus' kernel tree