include/asm-s390/bitops.h

   1 #ifndef _S390_BITOPS_H
   2 #define _S390_BITOPS_H
   3
   4 /*
   5  *  include/asm-s390/bitops.h
   6  *
   7  *  S390 version
   8  *    Copyright (C) 1999 IBM Deutschland Entwicklung GmbH, IBM Corporation
   9  *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
  10  *
  11  *  Derived from "include/asm-i386/bitops.h"
  12  *    Copyright (C) 1992, Linus Torvalds
  13  *
  14  */
  15 #include <linux/compiler.h>
  16
  17 /*
  18  * 32 bit bitops format:
  19  * bit 0 is the LSB of *addr; bit 31 is the MSB of *addr;
  20  * bit 32 is the LSB of *(addr+4). That combined with the
  21  * big endian byte order on S390 give the following bit
  22  * order in memory:
  23  *    1f 1e 1d 1c 1b 1a 19 18 17 16 15 14 13 12 11 10 \
  24  *    0f 0e 0d 0c 0b 0a 09 08 07 06 05 04 03 02 01 00
  25  * after that follows the next long with bit numbers
  26  *    3f 3e 3d 3c 3b 3a 39 38 37 36 35 34 33 32 31 30
  27  *    2f 2e 2d 2c 2b 2a 29 28 27 26 25 24 23 22 21 20
  28  * The reason for this bit ordering is the fact that
  29  * in the architecture independent code bits operations
  30  * of the form "flags |= (1 << bitnr)" are used INTERMIXED
  31  * with operation of the form "set_bit(bitnr, flags)".
  32  *
  33  * 64 bit bitops format:
  34  * bit 0 is the LSB of *addr; bit 63 is the MSB of *addr;
  35  * bit 64 is the LSB of *(addr+8). That combined with the
  36  * big endian byte order on S390 give the following bit
  37  * order in memory:
  38  *    3f 3e 3d 3c 3b 3a 39 38 37 36 35 34 33 32 31 30
  39  *    2f 2e 2d 2c 2b 2a 29 28 27 26 25 24 23 22 21 20
  40  *    1f 1e 1d 1c 1b 1a 19 18 17 16 15 14 13 12 11 10
  41  *    0f 0e 0d 0c 0b 0a 09 08 07 06 05 04 03 02 01 00
  42  * after that follows the next long with bit numbers
  43  *    7f 7e 7d 7c 7b 7a 79 78 77 76 75 74 73 72 71 70
  44  *    6f 6e 6d 6c 6b 6a 69 68 67 66 65 64 63 62 61 60
  45  *    5f 5e 5d 5c 5b 5a 59 58 57 56 55 54 53 52 51 50
  46  *    4f 4e 4d 4c 4b 4a 49 48 47 46 45 44 43 42 41 40
  47  * The reason for this bit ordering is the fact that
  48  * in the architecture independent code bits operations
  49  * of the form "flags |= (1 << bitnr)" are used INTERMIXED
  50  * with operation of the form "set_bit(bitnr, flags)".
  51  */
  52
  53 /* set ALIGN_CS to 1 if the SMP safe bit operations should
  54  * align the address to 4 byte boundary. It seems to work
  55  * without the alignment.
  56  */
  57 #ifdef __KERNEL__
  58 #define ALIGN_CS 0
  59 #else
  60 #define ALIGN_CS 1
  61 #ifndef CONFIG_SMP
  62 #error "bitops won't work without CONFIG_SMP"
  63 #endif
  64 #endif
  65
  66 /* bitmap tables from arch/S390/kernel/bitmap.S */
  67 extern const char _oi_bitmap[];
  68 extern const char _ni_bitmap[];
  69 extern const char _zb_findmap[];
  70 extern const char _sb_findmap[];
  71
  72 #ifndef __s390x__
  73
  74 #define __BITOPS_ALIGN          3
  75 #define __BITOPS_WORDSIZE       32
  76 #define __BITOPS_OR             "or"
  77 #define __BITOPS_AND            "nr"
  78 #define __BITOPS_XOR            "xr"
  79
  80 #define __BITOPS_LOOP(__old, __new, __addr, __val, __op_string)         \
  81         __asm__ __volatile__("   l   %0,0(%4)\n"                        \
  82                              "0: lr  %1,%0\n"                           \
  83                              __op_string "  %1,%3\n"                    \
  84                              "   cs  %0,%1,0(%4)\n"                     \
  85                              "   jl  0b"                                \
  86                              : "=&d" (__old), "=&d" (__new),            \
  87                                "=m" (*(unsigned long *) __addr)         \
  88                              : "d" (__val), "a" (__addr),               \
  89                                "m" (*(unsigned long *) __addr) : "cc" );
  90
  91 #else /* __s390x__ */
  92
  93 #define __BITOPS_ALIGN          7
  94 #define __BITOPS_WORDSIZE       64
  95 #define __BITOPS_OR             "ogr"
  96 #define __BITOPS_AND            "ngr"
  97 #define __BITOPS_XOR            "xgr"
  98
  99 #define __BITOPS_LOOP(__old, __new, __addr, __val, __op_string)         \
 100         __asm__ __volatile__("   lg  %0,0(%4)\n"                        \
 101                              "0: lgr %1,%0\n"                           \
 102                              __op_string "  %1,%3\n"                    \
 103                              "   csg %0,%1,0(%4)\n"                     \
 104                              "   jl  0b"                                \
 105                              : "=&d" (__old), "=&d" (__new),            \
 106                                "=m" (*(unsigned long *) __addr)         \
 107                              : "d" (__val), "a" (__addr),               \
 108                                "m" (*(unsigned long *) __addr) : "cc" );
 109
 110 #endif /* __s390x__ */
 111
 112 #define __BITOPS_WORDS(bits) (((bits)+__BITOPS_WORDSIZE-1)/__BITOPS_WORDSIZE)
 113 #define __BITOPS_BARRIER() __asm__ __volatile__ ( "" : : : "memory" )
 114
 115 #ifdef CONFIG_SMP
 116 /*
 117  * SMP safe set_bit routine based on compare and swap (CS)
 118  */
 119 static inline void set_bit_cs(unsigned long nr, volatile unsigned long *ptr)
 120 {
 121         unsigned long addr, old, new, mask;
 122
 123         addr = (unsigned long) ptr;
 124 #if ALIGN_CS == 1
 125         nr += (addr & __BITOPS_ALIGN) << 3;    /* add alignment to bit number */
 126         addr ^= addr & __BITOPS_ALIGN;         /* align address to 8 */
 127 #endif
 128         /* calculate address for CS */
 129         addr += (nr ^ (nr & (__BITOPS_WORDSIZE - 1))) >> 3;
 130         /* make OR mask */
 131         mask = 1UL << (nr & (__BITOPS_WORDSIZE - 1));
 132         /* Do the atomic update. */
 133         __BITOPS_LOOP(old, new, addr, mask, __BITOPS_OR);
 134 }
 135
 136 /*
 137  * SMP safe clear_bit routine based on compare and swap (CS)
 138  */
 139 static inline void clear_bit_cs(unsigned long nr, volatile unsigned long *ptr)
 140 {
 141         unsigned long addr, old, new, mask;
 142
 143         addr = (unsigned long) ptr;
 144 #if ALIGN_CS == 1
 145         nr += (addr & __BITOPS_ALIGN) << 3;    /* add alignment to bit number */
 146         addr ^= addr & __BITOPS_ALIGN;         /* align address to 8 */
 147 #endif
 148         /* calculate address for CS */
 149         addr += (nr ^ (nr & (__BITOPS_WORDSIZE - 1))) >> 3;
 150         /* make AND mask */
 151         mask = ~(1UL << (nr & (__BITOPS_WORDSIZE - 1)));
 152         /* Do the atomic update. */
 153         __BITOPS_LOOP(old, new, addr, mask, __BITOPS_AND);
 154 }
 155
 156 /*
 157  * SMP safe change_bit routine based on compare and swap (CS)
 158  */
 159 static inline void change_bit_cs(unsigned long nr, volatile unsigned long *ptr)
 160 {
 161         unsigned long addr, old, new, mask;
 162
 163         addr = (unsigned long) ptr;
 164 #if ALIGN_CS == 1
 165         nr += (addr & __BITOPS_ALIGN) << 3;    /* add alignment to bit number */
 166         addr ^= addr & __BITOPS_ALIGN;         /* align address to 8 */
 167 #endif
 168         /* calculate address for CS */
 169         addr += (nr ^ (nr & (__BITOPS_WORDSIZE - 1))) >> 3;
 170         /* make XOR mask */
 171         mask = 1UL << (nr & (__BITOPS_WORDSIZE - 1));
 172         /* Do the atomic update. */
 173         __BITOPS_LOOP(old, new, addr, mask, __BITOPS_XOR);
 174 }
 175
 176 /*
 177  * SMP safe test_and_set_bit routine based on compare and swap (CS)
 178  */
 179 static inline int
 180 test_and_set_bit_cs(unsigned long nr, volatile unsigned long *ptr)
 181 {
 182         unsigned long addr, old, new, mask;
 183
 184         addr = (unsigned long) ptr;
 185 #if ALIGN_CS == 1
 186         nr += (addr & __BITOPS_ALIGN) << 3;    /* add alignment to bit number */
 187         addr ^= addr & __BITOPS_ALIGN;         /* align address to 8 */
 188 #endif
 189         /* calculate address for CS */
 190         addr += (nr ^ (nr & (__BITOPS_WORDSIZE - 1))) >> 3;
 191         /* make OR/test mask */
 192         mask = 1UL << (nr & (__BITOPS_WORDSIZE - 1));
 193         /* Do the atomic update. */
 194         __BITOPS_LOOP(old, new, addr, mask, __BITOPS_OR);
 195         __BITOPS_BARRIER();
 196         return (old & mask) != 0;
 197 }
 198
 199 /*
 200  * SMP safe test_and_clear_bit routine based on compare and swap (CS)
 201  */
 202 static inline int
 203 test_and_clear_bit_cs(unsigned long nr, volatile unsigned long *ptr)
 204 {
 205         unsigned long addr, old, new, mask;
 206
 207         addr = (unsigned long) ptr;
 208 #if ALIGN_CS == 1
 209         nr += (addr & __BITOPS_ALIGN) << 3;    /* add alignment to bit number */
 210         addr ^= addr & __BITOPS_ALIGN;         /* align address to 8 */
 211 #endif
 212         /* calculate address for CS */
 213         addr += (nr ^ (nr & (__BITOPS_WORDSIZE - 1))) >> 3;
 214         /* make AND/test mask */
 215         mask = ~(1UL << (nr & (__BITOPS_WORDSIZE - 1)));
 216         /* Do the atomic update. */
 217         __BITOPS_LOOP(old, new, addr, mask, __BITOPS_AND);
 218         __BITOPS_BARRIER();
 219         return (old ^ new) != 0;
 220 }
 221
 222 /*
 223  * SMP safe test_and_change_bit routine based on compare and swap (CS)
 224  */
 225 static inline int
 226 test_and_change_bit_cs(unsigned long nr, volatile unsigned long *ptr)
 227 {
 228         unsigned long addr, old, new, mask;
 229
 230         addr = (unsigned long) ptr;
 231 #if ALIGN_CS == 1
 232         nr += (addr & __BITOPS_ALIGN) << 3;  /* add alignment to bit number */
 233         addr ^= addr & __BITOPS_ALIGN;       /* align address to 8 */
 234 #endif
 235         /* calculate address for CS */
 236         addr += (nr ^ (nr & (__BITOPS_WORDSIZE - 1))) >> 3;
 237         /* make XOR/test mask */
 238         mask = 1UL << (nr & (__BITOPS_WORDSIZE - 1));
 239         /* Do the atomic update. */
 240         __BITOPS_LOOP(old, new, addr, mask, __BITOPS_XOR);
 241         __BITOPS_BARRIER();
 242         return (old & mask) != 0;
 243 }
 244 #endif /* CONFIG_SMP */
 245
 246 /*
 247  * fast, non-SMP set_bit routine
 248  */
 249 static inline void __set_bit(unsigned long nr, volatile unsigned long *ptr)
 250 {
 251         unsigned long addr;
 252
 253         addr = (unsigned long) ptr + ((nr ^ (__BITOPS_WORDSIZE - 8)) >> 3);
 254         asm volatile("oc 0(1,%1),0(%2)"
 255                      : "=m" (*(char *) addr)
 256                      : "a" (addr), "a" (_oi_bitmap + (nr & 7)),
 257                        "m" (*(char *) addr) : "cc" );
 258 }
 259
 260 static inline void
 261 __constant_set_bit(const unsigned long nr, volatile unsigned long *ptr)
 262 {
 263         unsigned long addr;
 264
 265         addr = ((unsigned long) ptr) + ((nr ^ (__BITOPS_WORDSIZE - 8)) >> 3);
 266         switch (nr&7) {
 267         case 0:
 268                 asm volatile ("oi 0(%1),0x01" : "=m" (*(char *) addr)
 269                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 270                 break;
 271         case 1:
 272                 asm volatile ("oi 0(%1),0x02" : "=m" (*(char *) addr)
 273                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 274                 break;
 275         case 2:
 276                 asm volatile ("oi 0(%1),0x04" : "=m" (*(char *) addr)
 277                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 278                 break;
 279         case 3:
 280                 asm volatile ("oi 0(%1),0x08" : "=m" (*(char *) addr)
 281                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 282                 break;
 283         case 4:
 284                 asm volatile ("oi 0(%1),0x10" : "=m" (*(char *) addr)
 285                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 286                 break;
 287         case 5:
 288                 asm volatile ("oi 0(%1),0x20" : "=m" (*(char *) addr)
 289                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 290                 break;
 291         case 6:
 292                 asm volatile ("oi 0(%1),0x40" : "=m" (*(char *) addr)
 293                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 294                 break;
 295         case 7:
 296                 asm volatile ("oi 0(%1),0x80" : "=m" (*(char *) addr)
 297                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 298                 break;
 299         }
 300 }
 301
 302 #define set_bit_simple(nr,addr) \
 303 (__builtin_constant_p((nr)) ? \
 304  __constant_set_bit((nr),(addr)) : \
 305  __set_bit((nr),(addr)) )
 306
 307 /*
 308  * fast, non-SMP clear_bit routine
 309  */
 310 static inline void
 311 __clear_bit(unsigned long nr, volatile unsigned long *ptr)
 312 {
 313         unsigned long addr;
 314
 315         addr = (unsigned long) ptr + ((nr ^ (__BITOPS_WORDSIZE - 8)) >> 3);
 316         asm volatile("nc 0(1,%1),0(%2)"
 317                      : "=m" (*(char *) addr)
 318                      : "a" (addr), "a" (_ni_bitmap + (nr & 7)),
 319                        "m" (*(char *) addr) : "cc" );
 320 }
 321
 322 static inline void
 323 __constant_clear_bit(const unsigned long nr, volatile unsigned long *ptr)
 324 {
 325         unsigned long addr;
 326
 327         addr = ((unsigned long) ptr) + ((nr ^ (__BITOPS_WORDSIZE - 8)) >> 3);
 328         switch (nr&7) {
 329         case 0:
 330                 asm volatile ("ni 0(%1),0xFE" : "=m" (*(char *) addr)
 331                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 332                 break;
 333         case 1:
 334                 asm volatile ("ni 0(%1),0xFD": "=m" (*(char *) addr)
 335                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 336                 break;
 337         case 2:
 338                 asm volatile ("ni 0(%1),0xFB" : "=m" (*(char *) addr)
 339                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 340                 break;
 341         case 3:
 342                 asm volatile ("ni 0(%1),0xF7" : "=m" (*(char *) addr)
 343                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 344                 break;
 345         case 4:
 346                 asm volatile ("ni 0(%1),0xEF" : "=m" (*(char *) addr)
 347                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 348                 break;
 349         case 5:
 350                 asm volatile ("ni 0(%1),0xDF" : "=m" (*(char *) addr)
 351                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 352                 break;
 353         case 6:
 354                 asm volatile ("ni 0(%1),0xBF" : "=m" (*(char *) addr)
 355                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 356                 break;
 357         case 7:
 358                 asm volatile ("ni 0(%1),0x7F" : "=m" (*(char *) addr)
 359                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 360                 break;
 361         }
 362 }
 363
 364 #define clear_bit_simple(nr,addr) \
 365 (__builtin_constant_p((nr)) ? \
 366  __constant_clear_bit((nr),(addr)) : \
 367  __clear_bit((nr),(addr)) )
 368
 369 /*
 370  * fast, non-SMP change_bit routine
 371  */
 372 static inline void __change_bit(unsigned long nr, volatile unsigned long *ptr)
 373 {
 374         unsigned long addr;
 375
 376         addr = (unsigned long) ptr + ((nr ^ (__BITOPS_WORDSIZE - 8)) >> 3);
 377         asm volatile("xc 0(1,%1),0(%2)"
 378                      :  "=m" (*(char *) addr)
 379                      : "a" (addr), "a" (_oi_bitmap + (nr & 7)),
 380                        "m" (*(char *) addr) : "cc" );
 381 }
 382
 383 static inline void
 384 __constant_change_bit(const unsigned long nr, volatile unsigned long *ptr)
 385 {
 386         unsigned long addr;
 387
 388         addr = ((unsigned long) ptr) + ((nr ^ (__BITOPS_WORDSIZE - 8)) >> 3);
 389         switch (nr&7) {
 390         case 0:
 391                 asm volatile ("xi 0(%1),0x01" : "=m" (*(char *) addr)
 392                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 393                 break;
 394         case 1:
 395                 asm volatile ("xi 0(%1),0x02" : "=m" (*(char *) addr)
 396                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 397                 break;
 398         case 2:
 399                 asm volatile ("xi 0(%1),0x04" : "=m" (*(char *) addr)
 400                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 401                 break;
 402         case 3:
 403                 asm volatile ("xi 0(%1),0x08" : "=m" (*(char *) addr)
 404                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 405                 break;
 406         case 4:
 407                 asm volatile ("xi 0(%1),0x10" : "=m" (*(char *) addr)
 408                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 409                 break;
 410         case 5:
 411                 asm volatile ("xi 0(%1),0x20" : "=m" (*(char *) addr)
 412                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 413                 break;
 414         case 6:
 415                 asm volatile ("xi 0(%1),0x40" : "=m" (*(char *) addr)
 416                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 417                 break;
 418         case 7:
 419                 asm volatile ("xi 0(%1),0x80" : "=m" (*(char *) addr)
 420                               : "a" (addr), "m" (*(char *) addr) : "cc" );
 421                 break;
 422         }
 423 }
 424
 425 #define change_bit_simple(nr,addr) \
 426 (__builtin_constant_p((nr)) ? \
 427  __constant_change_bit((nr),(addr)) : \
 428  __change_bit((nr),(addr)) )
 429
 430 /*
 431  * fast, non-SMP test_and_set_bit routine
 432  */
 433 static inline int
 434 test_and_set_bit_simple(unsigned long nr, volatile unsigned long *ptr)
 435 {
 436         unsigned long addr;
 437         unsigned char ch;
 438
 439         addr = (unsigned long) ptr + ((nr ^ (__BITOPS_WORDSIZE - 8)) >> 3);
 440         ch = *(unsigned char *) addr;
 441         asm volatile("oc 0(1,%1),0(%2)"
 442                      : "=m" (*(char *) addr)
 443                      : "a" (addr), "a" (_oi_bitmap + (nr & 7)),
 444                        "m" (*(char *) addr) : "cc", "memory" );
 445         return (ch >> (nr & 7)) & 1;
 446 }
 447 #define __test_and_set_bit(X,Y)         test_and_set_bit_simple(X,Y)
 448
 449 /*
 450  * fast, non-SMP test_and_clear_bit routine
 451  */
 452 static inline int
 453 test_and_clear_bit_simple(unsigned long nr, volatile unsigned long *ptr)
 454 {
 455         unsigned long addr;
 456         unsigned char ch;
 457
 458         addr = (unsigned long) ptr + ((nr ^ (__BITOPS_WORDSIZE - 8)) >> 3);
 459         ch = *(unsigned char *) addr;
 460         asm volatile("nc 0(1,%1),0(%2)"
 461                      : "=m" (*(char *) addr)
 462                      : "a" (addr), "a" (_ni_bitmap + (nr & 7)),
 463                        "m" (*(char *) addr) : "cc", "memory" );
 464         return (ch >> (nr & 7)) & 1;
 465 }
 466 #define __test_and_clear_bit(X,Y)       test_and_clear_bit_simple(X,Y)
 467
 468 /*
 469  * fast, non-SMP test_and_change_bit routine
 470  */
 471 static inline int
 472 test_and_change_bit_simple(unsigned long nr, volatile unsigned long *ptr)
 473 {
 474         unsigned long addr;
 475         unsigned char ch;
 476
 477         addr = (unsigned long) ptr + ((nr ^ (__BITOPS_WORDSIZE - 8)) >> 3);
 478         ch = *(unsigned char *) addr;
 479         asm volatile("xc 0(1,%1),0(%2)"
 480                      : "=m" (*(char *) addr)
 481                      : "a" (addr), "a" (_oi_bitmap + (nr & 7)),
 482                        "m" (*(char *) addr) : "cc", "memory" );
 483         return (ch >> (nr & 7)) & 1;
 484 }
 485 #define __test_and_change_bit(X,Y)      test_and_change_bit_simple(X,Y)
 486
 487 #ifdef CONFIG_SMP
 488 #define set_bit             set_bit_cs
 489 #define clear_bit           clear_bit_cs
 490 #define change_bit          change_bit_cs
 491 #define test_and_set_bit    test_and_set_bit_cs
 492 #define test_and_clear_bit  test_and_clear_bit_cs
 493 #define test_and_change_bit test_and_change_bit_cs
 494 #else
 495 #define set_bit             set_bit_simple
 496 #define clear_bit           clear_bit_simple
 497 #define change_bit          change_bit_simple
 498 #define test_and_set_bit    test_and_set_bit_simple
 499 #define test_and_clear_bit  test_and_clear_bit_simple
 500 #define test_and_change_bit test_and_change_bit_simple
 501 #endif
 502
 503
 504 /*
 505  * This routine doesn't need to be atomic.
 506  */
 507
 508 static inline int __test_bit(unsigned long nr, const volatile unsigned long *ptr)
 509 {
 510         unsigned long addr;
 511         unsigned char ch;
 512
 513         addr = (unsigned long) ptr + ((nr ^ (__BITOPS_WORDSIZE - 8)) >> 3);
 514         ch = *(volatile unsigned char *) addr;
 515         return (ch >> (nr & 7)) & 1;
 516 }
 517
 518 static inline int
 519 __constant_test_bit(unsigned long nr, const volatile unsigned long *addr) {
 520     return (((volatile char *) addr)
 521             [(nr^(__BITOPS_WORDSIZE-8))>>3] & (1<<(nr&7))) != 0;
 522 }
 523
 524 #define test_bit(nr,addr) \
 525 (__builtin_constant_p((nr)) ? \
 526  __constant_test_bit((nr),(addr)) : \
 527  __test_bit((nr),(addr)) )
 528
 529 /*
 530  * ffz = Find First Zero in word. Undefined if no zero exists,
 531  * so code should check against ~0UL first..
 532  */
 533 static inline unsigned long ffz(unsigned long word)
 534 {
 535         unsigned long bit = 0;
 536
 537 #ifdef __s390x__
 538         if (likely((word & 0xffffffff) == 0xffffffff)) {
 539                 word >>= 32;
 540                 bit += 32;
 541         }
 542 #endif
 543         if (likely((word & 0xffff) == 0xffff)) {
 544                 word >>= 16;
 545                 bit += 16;
 546         }
 547         if (likely((word & 0xff) == 0xff)) {
 548                 word >>= 8;
 549                 bit += 8;
 550         }
 551         return bit + _zb_findmap[word & 0xff];
 552 }
 553
 554 /*
 555  * __ffs = find first bit in word. Undefined if no bit exists,
 556  * so code should check against 0UL first..
 557  */
 558 static inline unsigned long __ffs (unsigned long word)
 559 {
 560         unsigned long bit = 0;
 561
 562 #ifdef __s390x__
 563         if (likely((word & 0xffffffff) == 0)) {
 564                 word >>= 32;
 565                 bit += 32;
 566         }
 567 #endif
 568         if (likely((word & 0xffff) == 0)) {
 569                 word >>= 16;
 570                 bit += 16;
 571         }
 572         if (likely((word & 0xff) == 0)) {
 573                 word >>= 8;
 574                 bit += 8;
 575         }
 576         return bit + _sb_findmap[word & 0xff];
 577 }
 578
 579 /*
 580  * Find-bit routines..
 581  */
 582
 583 #ifndef __s390x__
 584
 585 static inline int
 586 find_first_zero_bit(const unsigned long * addr, unsigned long size)
 587 {
 588         typedef struct { long _[__BITOPS_WORDS(size)]; } addrtype;
 589         unsigned long cmp, count;
 590         unsigned int res;
 591
 592         if (!size)
 593                 return 0;
 594         __asm__("   lhi  %1,-1\n"
 595                 "   lr   %2,%3\n"
 596                 "   slr  %0,%0\n"
 597                 "   ahi  %2,31\n"
 598                 "   srl  %2,5\n"
 599                 "0: c    %1,0(%0,%4)\n"
 600                 "   jne  1f\n"
 601                 "   la   %0,4(%0)\n"
 602                 "   brct %2,0b\n"
 603                 "   lr   %0,%3\n"
 604                 "   j    4f\n"
 605                 "1: l    %2,0(%0,%4)\n"
 606                 "   sll  %0,3\n"
 607                 "   lhi  %1,0xff\n"
 608                 "   tml  %2,0xffff\n"
 609                 "   jno  2f\n"
 610                 "   ahi  %0,16\n"
 611                 "   srl  %2,16\n"
 612                 "2: tml  %2,0x00ff\n"
 613                 "   jno  3f\n"
 614                 "   ahi  %0,8\n"
 615                 "   srl  %2,8\n"
 616                 "3: nr   %2,%1\n"
 617                 "   ic   %2,0(%2,%5)\n"
 618                 "   alr  %0,%2\n"
 619                 "4:"
 620                 : "=&a" (res), "=&d" (cmp), "=&a" (count)
 621                 : "a" (size), "a" (addr), "a" (&_zb_findmap),
 622                   "m" (*(addrtype *) addr) : "cc" );
 623         return (res < size) ? res : size;
 624 }
 625
 626 static inline int
 627 find_first_bit(const unsigned long * addr, unsigned long size)
 628 {
 629         typedef struct { long _[__BITOPS_WORDS(size)]; } addrtype;
 630         unsigned long cmp, count;
 631         unsigned int res;
 632
 633         if (!size)
 634                 return 0;
 635         __asm__("   slr  %1,%1\n"
 636                 "   lr   %2,%3\n"
 637                 "   slr  %0,%0\n"
 638                 "   ahi  %2,31\n"
 639                 "   srl  %2,5\n"
 640                 "0: c    %1,0(%0,%4)\n"
 641                 "   jne  1f\n"
 642                 "   la   %0,4(%0)\n"
 643                 "   brct %2,0b\n"
 644                 "   lr   %0,%3\n"
 645                 "   j    4f\n"
 646                 "1: l    %2,0(%0,%4)\n"
 647                 "   sll  %0,3\n"
 648                 "   lhi  %1,0xff\n"
 649                 "   tml  %2,0xffff\n"
 650                 "   jnz  2f\n"
 651                 "   ahi  %0,16\n"
 652                 "   srl  %2,16\n"
 653                 "2: tml  %2,0x00ff\n"
 654                 "   jnz  3f\n"
 655                 "   ahi  %0,8\n"
 656                 "   srl  %2,8\n"
 657                 "3: nr   %2,%1\n"
 658                 "   ic   %2,0(%2,%5)\n"
 659                 "   alr  %0,%2\n"
 660                 "4:"
 661                 : "=&a" (res), "=&d" (cmp), "=&a" (count)
 662                 : "a" (size), "a" (addr), "a" (&_sb_findmap),
 663                   "m" (*(addrtype *) addr) : "cc" );
 664         return (res < size) ? res : size;
 665 }
 666
 667 #else /* __s390x__ */
 668
 669 static inline unsigned long
 670 find_first_zero_bit(const unsigned long * addr, unsigned long size)
 671 {
 672         typedef struct { long _[__BITOPS_WORDS(size)]; } addrtype;
 673         unsigned long res, cmp, count;
 674
 675         if (!size)
 676                 return 0;
 677         __asm__("   lghi  %1,-1\n"
 678                 "   lgr   %2,%3\n"
 679                 "   slgr  %0,%0\n"
 680                 "   aghi  %2,63\n"
 681                 "   srlg  %2,%2,6\n"
 682                 "0: cg    %1,0(%0,%4)\n"
 683                 "   jne   1f\n"
 684                 "   la    %0,8(%0)\n"
 685                 "   brct  %2,0b\n"
 686                 "   lgr   %0,%3\n"
 687                 "   j     5f\n"
 688                 "1: lg    %2,0(%0,%4)\n"
 689                 "   sllg  %0,%0,3\n"
 690                 "   clr   %2,%1\n"
 691                 "   jne   2f\n"
 692                 "   aghi  %0,32\n"
 693                 "   srlg  %2,%2,32\n"
 694                 "2: lghi  %1,0xff\n"
 695                 "   tmll  %2,0xffff\n"
 696                 "   jno   3f\n"
 697                 "   aghi  %0,16\n"
 698                 "   srl   %2,16\n"
 699                 "3: tmll  %2,0x00ff\n"
 700                 "   jno   4f\n"
 701                 "   aghi  %0,8\n"
 702                 "   srl   %2,8\n"
 703                 "4: ngr   %2,%1\n"
 704                 "   ic    %2,0(%2,%5)\n"
 705                 "   algr  %0,%2\n"
 706                 "5:"
 707                 : "=&a" (res), "=&d" (cmp), "=&a" (count)
 708                 : "a" (size), "a" (addr), "a" (&_zb_findmap),
 709                   "m" (*(addrtype *) addr) : "cc" );
 710         return (res < size) ? res : size;
 711 }
 712
 713 static inline unsigned long
 714 find_first_bit(const unsigned long * addr, unsigned long size)
 715 {
 716         typedef struct { long _[__BITOPS_WORDS(size)]; } addrtype;
 717         unsigned long res, cmp, count;
 718
 719         if (!size)
 720                 return 0;
 721         __asm__("   slgr  %1,%1\n"
 722                 "   lgr   %2,%3\n"
 723                 "   slgr  %0,%0\n"
 724                 "   aghi  %2,63\n"
 725                 "   srlg  %2,%2,6\n"
 726                 "0: cg    %1,0(%0,%4)\n"
 727                 "   jne   1f\n"
 728                 "   aghi  %0,8\n"
 729                 "   brct  %2,0b\n"
 730                 "   lgr   %0,%3\n"
 731                 "   j     5f\n"
 732                 "1: lg    %2,0(%0,%4)\n"
 733                 "   sllg  %0,%0,3\n"
 734                 "   clr   %2,%1\n"
 735                 "   jne   2f\n"
 736                 "   aghi  %0,32\n"
 737                 "   srlg  %2,%2,32\n"
 738                 "2: lghi  %1,0xff\n"
 739                 "   tmll  %2,0xffff\n"
 740                 "   jnz   3f\n"
 741                 "   aghi  %0,16\n"
 742                 "   srl   %2,16\n"
 743                 "3: tmll  %2,0x00ff\n"
 744                 "   jnz   4f\n"
 745                 "   aghi  %0,8\n"
 746                 "   srl   %2,8\n"
 747                 "4: ngr   %2,%1\n"
 748                 "   ic    %2,0(%2,%5)\n"
 749                 "   algr  %0,%2\n"
 750                 "5:"
 751                 : "=&a" (res), "=&d" (cmp), "=&a" (count)
 752                 : "a" (size), "a" (addr), "a" (&_sb_findmap),
 753                   "m" (*(addrtype *) addr) : "cc" );
 754         return (res < size) ? res : size;
 755 }
 756
 757 #endif /* __s390x__ */
 758
 759 static inline int
 760 find_next_zero_bit (const unsigned long * addr, unsigned long size,
 761                     unsigned long offset)
 762 {
 763         const unsigned long *p;
 764         unsigned long bit, set;
 765
 766         if (offset >= size)
 767                 return size;
 768         bit = offset & (__BITOPS_WORDSIZE - 1);
 769         offset -= bit;
 770         size -= offset;
 771         p = addr + offset / __BITOPS_WORDSIZE;
 772         if (bit) {
 773                 /*
 774                  * s390 version of ffz returns __BITOPS_WORDSIZE
 775                  * if no zero bit is present in the word.
 776                  */
 777                 set = ffz(*p >> bit) + bit;
 778                 if (set >= size)
 779                         return size + offset;
 780                 if (set < __BITOPS_WORDSIZE)
 781                         return set + offset;
 782                 offset += __BITOPS_WORDSIZE;
 783                 size -= __BITOPS_WORDSIZE;
 784                 p++;
 785         }
 786         return offset + find_first_zero_bit(p, size);
 787 }
 788
 789 static inline int
 790 find_next_bit (const unsigned long * addr, unsigned long size,
 791                unsigned long offset)
 792 {
 793         const unsigned long *p;
 794         unsigned long bit, set;
 795
 796         if (offset >= size)
 797                 return size;
 798         bit = offset & (__BITOPS_WORDSIZE - 1);
 799         offset -= bit;
 800         size -= offset;
 801         p = addr + offset / __BITOPS_WORDSIZE;
 802         if (bit) {
 803                 /*
 804                  * s390 version of __ffs returns __BITOPS_WORDSIZE
 805                  * if no one bit is present in the word.
 806                  */
 807                 set = __ffs(*p & (~0UL << bit));
 808                 if (set >= size)
 809                         return size + offset;
 810                 if (set < __BITOPS_WORDSIZE)
 811                         return set + offset;
 812                 offset += __BITOPS_WORDSIZE;
 813                 size -= __BITOPS_WORDSIZE;
 814                 p++;
 815         }
 816         return offset + find_first_bit(p, size);
 817 }
 818
 819 /*
 820  * Every architecture must define this function. It's the fastest
 821  * way of searching a 140-bit bitmap where the first 100 bits are
 822  * unlikely to be set. It's guaranteed that at least one of the 140
 823  * bits is cleared.
 824  */
 825 static inline int sched_find_first_bit(unsigned long *b)
 826 {
 827         return find_first_bit(b, 140);
 828 }
 829
 830 #include <asm-generic/bitops/ffs.h>
 831
 832 #include <asm-generic/bitops/fls.h>
 833 #include <asm-generic/bitops/fls64.h>
 834
 835 #include <asm-generic/bitops/hweight.h>
 836
 837 #ifdef __KERNEL__
 838
 839 /*
 840  * ATTENTION: intel byte ordering convention for ext2 and minix !!
 841  * bit 0 is the LSB of addr; bit 31 is the MSB of addr;
 842  * bit 32 is the LSB of (addr+4).
 843  * That combined with the little endian byte order of Intel gives the
 844  * following bit order in memory:
 845  *    07 06 05 04 03 02 01 00 15 14 13 12 11 10 09 08 \
 846  *    23 22 21 20 19 18 17 16 31 30 29 28 27 26 25 24
 847  */
 848
 849 #define ext2_set_bit(nr, addr)       \
 850         __test_and_set_bit((nr)^(__BITOPS_WORDSIZE - 8), (unsigned long *)addr)
 851 #define ext2_set_bit_atomic(lock, nr, addr)       \
 852         test_and_set_bit((nr)^(__BITOPS_WORDSIZE - 8), (unsigned long *)addr)
 853 #define ext2_clear_bit(nr, addr)     \
 854         __test_and_clear_bit((nr)^(__BITOPS_WORDSIZE - 8), (unsigned long *)addr)
 855 #define ext2_clear_bit_atomic(lock, nr, addr)     \
 856         test_and_clear_bit((nr)^(__BITOPS_WORDSIZE - 8), (unsigned long *)addr)
 857 #define ext2_test_bit(nr, addr)      \
 858         test_bit((nr)^(__BITOPS_WORDSIZE - 8), (unsigned long *)addr)
 859
 860 #ifndef __s390x__
 861
 862 static inline int
 863 ext2_find_first_zero_bit(void *vaddr, unsigned int size)
 864 {
 865         typedef struct { long _[__BITOPS_WORDS(size)]; } addrtype;
 866         unsigned long cmp, count;
 867         unsigned int res;
 868
 869         if (!size)
 870                 return 0;
 871         __asm__("   lhi  %1,-1\n"
 872                 "   lr   %2,%3\n"
 873                 "   ahi  %2,31\n"
 874                 "   srl  %2,5\n"
 875                 "   slr  %0,%0\n"
 876                 "0: cl   %1,0(%0,%4)\n"
 877                 "   jne  1f\n"
 878                 "   ahi  %0,4\n"
 879                 "   brct %2,0b\n"
 880                 "   lr   %0,%3\n"
 881                 "   j    4f\n"
 882                 "1: l    %2,0(%0,%4)\n"
 883                 "   sll  %0,3\n"
 884                 "   ahi  %0,24\n"
 885                 "   lhi  %1,0xff\n"
 886                 "   tmh  %2,0xffff\n"
 887                 "   jo   2f\n"
 888                 "   ahi  %0,-16\n"
 889                 "   srl  %2,16\n"
 890                 "2: tml  %2,0xff00\n"
 891                 "   jo   3f\n"
 892                 "   ahi  %0,-8\n"
 893                 "   srl  %2,8\n"
 894                 "3: nr   %2,%1\n"
 895                 "   ic   %2,0(%2,%5)\n"
 896                 "   alr  %0,%2\n"
 897                 "4:"
 898                 : "=&a" (res), "=&d" (cmp), "=&a" (count)
 899                 : "a" (size), "a" (vaddr), "a" (&_zb_findmap),
 900                   "m" (*(addrtype *) vaddr) : "cc" );
 901         return (res < size) ? res : size;
 902 }
 903
 904 #else /* __s390x__ */
 905
 906 static inline unsigned long
 907 ext2_find_first_zero_bit(void *vaddr, unsigned long size)
 908 {
 909         typedef struct { long _[__BITOPS_WORDS(size)]; } addrtype;
 910         unsigned long res, cmp, count;
 911
 912         if (!size)
 913                 return 0;
 914         __asm__("   lghi  %1,-1\n"
 915                 "   lgr   %2,%3\n"
 916                 "   aghi  %2,63\n"
 917                 "   srlg  %2,%2,6\n"
 918                 "   slgr  %0,%0\n"
 919                 "0: clg   %1,0(%0,%4)\n"
 920                 "   jne   1f\n"
 921                 "   aghi  %0,8\n"
 922                 "   brct  %2,0b\n"
 923                 "   lgr   %0,%3\n"
 924                 "   j     5f\n"
 925                 "1: cl    %1,0(%0,%4)\n"
 926                 "   jne   2f\n"
 927                 "   aghi  %0,4\n"
 928                 "2: l     %2,0(%0,%4)\n"
 929                 "   sllg  %0,%0,3\n"
 930                 "   aghi  %0,24\n"
 931                 "   lghi  %1,0xff\n"
 932                 "   tmlh  %2,0xffff\n"
 933                 "   jo    3f\n"
 934                 "   aghi  %0,-16\n"
 935                 "   srl   %2,16\n"
 936                 "3: tmll  %2,0xff00\n"
 937                 "   jo    4f\n"
 938                 "   aghi  %0,-8\n"
 939                 "   srl   %2,8\n"
 940                 "4: ngr   %2,%1\n"
 941                 "   ic    %2,0(%2,%5)\n"
 942                 "   algr  %0,%2\n"
 943                 "5:"
 944                 : "=&a" (res), "=&d" (cmp), "=&a" (count)
 945                 : "a" (size), "a" (vaddr), "a" (&_zb_findmap),
 946                   "m" (*(addrtype *) vaddr) : "cc" );
 947         return (res < size) ? res : size;
 948 }
 949
 950 #endif /* __s390x__ */
 951
 952 static inline int
 953 ext2_find_next_zero_bit(void *vaddr, unsigned long size, unsigned long offset)
 954 {
 955         unsigned long *addr = vaddr, *p;
 956         unsigned long word, bit, set;
 957
 958         if (offset >= size)
 959                 return size;
 960         bit = offset & (__BITOPS_WORDSIZE - 1);
 961         offset -= bit;
 962         size -= offset;
 963         p = addr + offset / __BITOPS_WORDSIZE;
 964         if (bit) {
 965 #ifndef __s390x__
 966                 asm("   ic   %0,0(%1)\n"
 967                     "   icm  %0,2,1(%1)\n"
 968                     "   icm  %0,4,2(%1)\n"
 969                     "   icm  %0,8,3(%1)"
 970                     : "=&a" (word) : "a" (p), "m" (*p) : "cc" );
 971 #else
 972                 asm("   lrvg %0,%1" : "=a" (word) : "m" (*p) );
 973 #endif
 974                 /*
 975                  * s390 version of ffz returns __BITOPS_WORDSIZE
 976                  * if no zero bit is present in the word.
 977                  */
 978                 set = ffz(word >> bit) + bit;
 979                 if (set >= size)
 980                         return size + offset;
 981                 if (set < __BITOPS_WORDSIZE)
 982                         return set + offset;
 983                 offset += __BITOPS_WORDSIZE;
 984                 size -= __BITOPS_WORDSIZE;
 985                 p++;
 986         }
 987         return offset + ext2_find_first_zero_bit(p, size);
 988 }
 989
 990 #include <asm-generic/bitops/minix.h>
 991
 992 #endif /* __KERNEL__ */
 993
 994 #endif /* _S390_BITOPS_H */