arch/s390/lib/uaccess_mvcos.c

   1 /*
   2  *  arch/s390/lib/uaccess_mvcos.c
   3  *
   4  *  Optimized user space space access functions based on mvcos.
   5  *
   6  *    Copyright (C) IBM Corp. 2006
   7  *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
   8  *               Gerald Schaefer (gerald.schaefer@de.ibm.com)
   9  */
  10
  11 #include <linux/errno.h>
  12 #include <linux/mm.h>
  13 #include <asm/uaccess.h>
  14 #include <asm/futex.h>
  15 #include "uaccess.h"
  16
  17 #ifndef __s390x__
  18 #define AHI     "ahi"
  19 #define ALR     "alr"
  20 #define CLR     "clr"
  21 #define LHI     "lhi"
  22 #define SLR     "slr"
  23 #else
  24 #define AHI     "aghi"
  25 #define ALR     "algr"
  26 #define CLR     "clgr"
  27 #define LHI     "lghi"
  28 #define SLR     "slgr"
  29 #endif
  30
  31 static size_t copy_from_user_mvcos(size_t size, const void __user *ptr, void *x)
  32 {
  33         register unsigned long reg0 asm("0") = 0x81UL;
  34         unsigned long tmp1, tmp2;
  35
  36         tmp1 = -4096UL;
  37         asm volatile(
  38                 "0: .insn ss,0xc80000000000,0(%0,%2),0(%1),0\n"
  39                 "   jz    7f\n"
  40                 "1:"ALR"  %0,%3\n"
  41                 "  "SLR"  %1,%3\n"
  42                 "  "SLR"  %2,%3\n"
  43                 "   j     0b\n"
  44                 "2: la    %4,4095(%1)\n"/* %4 = ptr + 4095 */
  45                 "   nr    %4,%3\n"      /* %4 = (ptr + 4095) & -4096 */
  46                 "  "SLR"  %4,%1\n"
  47                 "  "CLR"  %0,%4\n"      /* copy crosses next page boundary? */
  48                 "   jnh   4f\n"
  49                 "3: .insn ss,0xc80000000000,0(%4,%2),0(%1),0\n"
  50                 "  "SLR"  %0,%4\n"
  51                 "  "ALR"  %2,%4\n"
  52                 "4:"LHI"  %4,-1\n"
  53                 "  "ALR"  %4,%0\n"      /* copy remaining size, subtract 1 */
  54                 "   bras  %3,6f\n"      /* memset loop */
  55                 "   xc    0(1,%2),0(%2)\n"
  56                 "5: xc    0(256,%2),0(%2)\n"
  57                 "   la    %2,256(%2)\n"
  58                 "6:"AHI"  %4,-256\n"
  59                 "   jnm   5b\n"
  60                 "   ex    %4,0(%3)\n"
  61                 "   j     8f\n"
  62                 "7:"SLR"  %0,%0\n"
  63                 "8: \n"
  64                 EX_TABLE(0b,2b) EX_TABLE(3b,4b)
  65                 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
  66                 : "d" (reg0) : "cc", "memory");
  67         return size;
  68 }
  69
  70 static size_t copy_from_user_mvcos_check(size_t size, const void __user *ptr, void *x)
  71 {
  72         if (size <= 256)
  73                 return copy_from_user_std(size, ptr, x);
  74         return copy_from_user_mvcos(size, ptr, x);
  75 }
  76
  77 static size_t copy_to_user_mvcos(size_t size, void __user *ptr, const void *x)
  78 {
  79         register unsigned long reg0 asm("0") = 0x810000UL;
  80         unsigned long tmp1, tmp2;
  81
  82         tmp1 = -4096UL;
  83         asm volatile(
  84                 "0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n"
  85                 "   jz    4f\n"
  86                 "1:"ALR"  %0,%3\n"
  87                 "  "SLR"  %1,%3\n"
  88                 "  "SLR"  %2,%3\n"
  89                 "   j     0b\n"
  90                 "2: la    %4,4095(%1)\n"/* %4 = ptr + 4095 */
  91                 "   nr    %4,%3\n"      /* %4 = (ptr + 4095) & -4096 */
  92                 "  "SLR"  %4,%1\n"
  93                 "  "CLR"  %0,%4\n"      /* copy crosses next page boundary? */
  94                 "   jnh   5f\n"
  95                 "3: .insn ss,0xc80000000000,0(%4,%1),0(%2),0\n"
  96                 "  "SLR"  %0,%4\n"
  97                 "   j     5f\n"
  98                 "4:"SLR"  %0,%0\n"
  99                 "5: \n"
 100                 EX_TABLE(0b,2b) EX_TABLE(3b,5b)
 101                 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
 102                 : "d" (reg0) : "cc", "memory");
 103         return size;
 104 }
 105
 106 static size_t copy_to_user_mvcos_check(size_t size, void __user *ptr,
 107                                        const void *x)
 108 {
 109         if (size <= 256)
 110                 return copy_to_user_std(size, ptr, x);
 111         return copy_to_user_mvcos(size, ptr, x);
 112 }
 113
 114 static size_t copy_in_user_mvcos(size_t size, void __user *to,
 115                                  const void __user *from)
 116 {
 117         register unsigned long reg0 asm("0") = 0x810081UL;
 118         unsigned long tmp1, tmp2;
 119
 120         tmp1 = -4096UL;
 121         /* FIXME: copy with reduced length. */
 122         asm volatile(
 123                 "0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n"
 124                 "   jz    2f\n"
 125                 "1:"ALR"  %0,%3\n"
 126                 "  "SLR"  %1,%3\n"
 127                 "  "SLR"  %2,%3\n"
 128                 "   j     0b\n"
 129                 "2:"SLR"  %0,%0\n"
 130                 "3: \n"
 131                 EX_TABLE(0b,3b)
 132                 : "+a" (size), "+a" (to), "+a" (from), "+a" (tmp1), "=a" (tmp2)
 133                 : "d" (reg0) : "cc", "memory");
 134         return size;
 135 }
 136
 137 static size_t clear_user_mvcos(size_t size, void __user *to)
 138 {
 139         register unsigned long reg0 asm("0") = 0x810000UL;
 140         unsigned long tmp1, tmp2;
 141
 142         tmp1 = -4096UL;
 143         asm volatile(
 144                 "0: .insn ss,0xc80000000000,0(%0,%1),0(%4),0\n"
 145                 "   jz    4f\n"
 146                 "1:"ALR"  %0,%2\n"
 147                 "  "SLR"  %1,%2\n"
 148                 "   j     0b\n"
 149                 "2: la    %3,4095(%1)\n"/* %4 = to + 4095 */
 150                 "   nr    %3,%2\n"      /* %4 = (to + 4095) & -4096 */
 151                 "  "SLR"  %3,%1\n"
 152                 "  "CLR"  %0,%3\n"      /* copy crosses next page boundary? */
 153                 "   jnh   5f\n"
 154                 "3: .insn ss,0xc80000000000,0(%3,%1),0(%4),0\n"
 155                 "  "SLR"  %0,%3\n"
 156                 "   j     5f\n"
 157                 "4:"SLR"  %0,%0\n"
 158                 "5: \n"
 159                 EX_TABLE(0b,2b) EX_TABLE(3b,5b)
 160                 : "+a" (size), "+a" (to), "+a" (tmp1), "=a" (tmp2)
 161                 : "a" (empty_zero_page), "d" (reg0) : "cc", "memory");
 162         return size;
 163 }
 164
 165 #ifdef CONFIG_S390_SWITCH_AMODE
 166 static size_t strnlen_user_mvcos(size_t count, const char __user *src)
 167 {
 168         char buf[256];
 169         int rc;
 170         size_t done, len, len_str;
 171
 172         done = 0;
 173         do {
 174                 len = min(count - done, (size_t) 256);
 175                 rc = uaccess.copy_from_user(len, src + done, buf);
 176                 if (unlikely(rc == len))
 177                         return 0;
 178                 len -= rc;
 179                 len_str = strnlen(buf, len);
 180                 done += len_str;
 181         } while ((len_str == len) && (done < count));
 182         return done + 1;
 183 }
 184
 185 static size_t strncpy_from_user_mvcos(size_t count, const char __user *src,
 186                                       char *dst)
 187 {
 188         int rc;
 189         size_t done, len, len_str;
 190
 191         done = 0;
 192         do {
 193                 len = min(count - done, (size_t) 4096);
 194                 rc = uaccess.copy_from_user(len, src + done, dst);
 195                 if (unlikely(rc == len))
 196                         return -EFAULT;
 197                 len -= rc;
 198                 len_str = strnlen(dst, len);
 199                 done += len_str;
 200         } while ((len_str == len) && (done < count));
 201         return done;
 202 }
 203 #endif /* CONFIG_S390_SWITCH_AMODE */
 204
 205 struct uaccess_ops uaccess_mvcos = {
 206         .copy_from_user = copy_from_user_mvcos_check,
 207         .copy_from_user_small = copy_from_user_std,
 208         .copy_to_user = copy_to_user_mvcos_check,
 209         .copy_to_user_small = copy_to_user_std,
 210         .copy_in_user = copy_in_user_mvcos,
 211         .clear_user = clear_user_mvcos,
 212         .strnlen_user = strnlen_user_std,
 213         .strncpy_from_user = strncpy_from_user_std,
 214         .futex_atomic_op = futex_atomic_op_std,
 215         .futex_atomic_cmpxchg = futex_atomic_cmpxchg_std,
 216 };
 217
 218 #ifdef CONFIG_S390_SWITCH_AMODE
 219 struct uaccess_ops uaccess_mvcos_switch = {
 220         .copy_from_user = copy_from_user_mvcos,
 221         .copy_from_user_small = copy_from_user_mvcos,
 222         .copy_to_user = copy_to_user_mvcos,
 223         .copy_to_user_small = copy_to_user_mvcos,
 224         .copy_in_user = copy_in_user_mvcos,
 225         .clear_user = clear_user_mvcos,
 226         .strnlen_user = strnlen_user_mvcos,
 227         .strncpy_from_user = strncpy_from_user_mvcos,
 228         .futex_atomic_op = futex_atomic_op_pt,
 229         .futex_atomic_cmpxchg = futex_atomic_cmpxchg_pt,
 230 };
 231 #endif