target-s390x/mem_helper.c

   1 /*
   2  *  S/390 memory access helper routines
   3  *
   4  *  Copyright (c) 2009 Ulrich Hecht
   5  *  Copyright (c) 2009 Alexander Graf
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2 of the License, or (at your option) any later version.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Lesser General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  19  */
  20
  21 #include "qemu/osdep.h"
  22 #include "cpu.h"
  23 #include "exec/helper-proto.h"
  24 #include "exec/cpu_ldst.h"
  25
  26 #if !defined(CONFIG_USER_ONLY)
  27 #include "hw/s390x/storage-keys.h"
  28 #endif
  29
  30 /*****************************************************************************/
  31 /* Softmmu support */
  32 #if !defined(CONFIG_USER_ONLY)
  33
  34 /* try to fill the TLB and return an exception if error. If retaddr is
  35    NULL, it means that the function was called in C code (i.e. not
  36    from generated code or from helper.c) */
  37 /* XXX: fix it to restore all registers */
  38 void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
  39               uintptr_t retaddr)
  40 {
  41     int ret;
  42
  43     ret = s390_cpu_handle_mmu_fault(cs, addr, is_write, mmu_idx);
  44     if (unlikely(ret != 0)) {
  45         if (likely(retaddr)) {
  46             /* now we have a real cpu fault */
  47             cpu_restore_state(cs, retaddr);
  48         }
  49         cpu_loop_exit(cs);
  50     }
  51 }
  52
  53 #endif
  54
  55 /* #define DEBUG_HELPER */
  56 #ifdef DEBUG_HELPER
  57 #define HELPER_LOG(x...) qemu_log(x)
  58 #else
  59 #define HELPER_LOG(x...)
  60 #endif
  61
  62 /* Reduce the length so that addr + len doesn't cross a page boundary.  */
  63 static inline uint64_t adj_len_to_page(uint64_t len, uint64_t addr)
  64 {
  65 #ifndef CONFIG_USER_ONLY
  66     if ((addr & ~TARGET_PAGE_MASK) + len - 1 >= TARGET_PAGE_SIZE) {
  67         return -addr & ~TARGET_PAGE_MASK;
  68     }
  69 #endif
  70     return len;
  71 }
  72
  73 static void fast_memset(CPUS390XState *env, uint64_t dest, uint8_t byte,
  74                         uint32_t l)
  75 {
  76     int mmu_idx = cpu_mmu_index(env, false);
  77
  78     while (l > 0) {
  79         void *p = tlb_vaddr_to_host(env, dest, MMU_DATA_STORE, mmu_idx);
  80         if (p) {
  81             /* Access to the whole page in write mode granted.  */
  82             int l_adj = adj_len_to_page(l, dest);
  83             memset(p, byte, l_adj);
  84             dest += l_adj;
  85             l -= l_adj;
  86         } else {
  87             /* We failed to get access to the whole page. The next write
  88                access will likely fill the QEMU TLB for the next iteration.  */
  89             cpu_stb_data(env, dest, byte);
  90             dest++;
  91             l--;
  92         }
  93     }
  94 }
  95
  96 static void fast_memmove(CPUS390XState *env, uint64_t dest, uint64_t src,
  97                          uint32_t l)
  98 {
  99     int mmu_idx = cpu_mmu_index(env, false);
 100
 101     while (l > 0) {
 102         void *src_p = tlb_vaddr_to_host(env, src, MMU_DATA_LOAD, mmu_idx);
 103         void *dest_p = tlb_vaddr_to_host(env, dest, MMU_DATA_STORE, mmu_idx);
 104         if (src_p && dest_p) {
 105             /* Access to both whole pages granted.  */
 106             int l_adj = adj_len_to_page(l, src);
 107             l_adj = adj_len_to_page(l_adj, dest);
 108             memmove(dest_p, src_p, l_adj);
 109             src += l_adj;
 110             dest += l_adj;
 111             l -= l_adj;
 112         } else {
 113             /* We failed to get access to one or both whole pages. The next
 114                read or write access will likely fill the QEMU TLB for the
 115                next iteration.  */
 116             cpu_stb_data(env, dest, cpu_ldub_data(env, src));
 117             src++;
 118             dest++;
 119             l--;
 120         }
 121     }
 122 }
 123
 124 /* and on array */
 125 uint32_t HELPER(nc)(CPUS390XState *env, uint32_t l, uint64_t dest,
 126                     uint64_t src)
 127 {
 128     int i;
 129     unsigned char x;
 130     uint32_t cc = 0;
 131
 132     HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
 133                __func__, l, dest, src);
 134     for (i = 0; i <= l; i++) {
 135         x = cpu_ldub_data(env, dest + i) & cpu_ldub_data(env, src + i);
 136         if (x) {
 137             cc = 1;
 138         }
 139         cpu_stb_data(env, dest + i, x);
 140     }
 141     return cc;
 142 }
 143
 144 /* xor on array */
 145 uint32_t HELPER(xc)(CPUS390XState *env, uint32_t l, uint64_t dest,
 146                     uint64_t src)
 147 {
 148     int i;
 149     unsigned char x;
 150     uint32_t cc = 0;
 151
 152     HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
 153                __func__, l, dest, src);
 154
 155     /* xor with itself is the same as memset(0) */
 156     if (src == dest) {
 157         fast_memset(env, dest, 0, l + 1);
 158         return 0;
 159     }
 160
 161     for (i = 0; i <= l; i++) {
 162         x = cpu_ldub_data(env, dest + i) ^ cpu_ldub_data(env, src + i);
 163         if (x) {
 164             cc = 1;
 165         }
 166         cpu_stb_data(env, dest + i, x);
 167     }
 168     return cc;
 169 }
 170
 171 /* or on array */
 172 uint32_t HELPER(oc)(CPUS390XState *env, uint32_t l, uint64_t dest,
 173                     uint64_t src)
 174 {
 175     int i;
 176     unsigned char x;
 177     uint32_t cc = 0;
 178
 179     HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
 180                __func__, l, dest, src);
 181     for (i = 0; i <= l; i++) {
 182         x = cpu_ldub_data(env, dest + i) | cpu_ldub_data(env, src + i);
 183         if (x) {
 184             cc = 1;
 185         }
 186         cpu_stb_data(env, dest + i, x);
 187     }
 188     return cc;
 189 }
 190
 191 /* memmove */
 192 void HELPER(mvc)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
 193 {
 194     int i = 0;
 195
 196     HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
 197                __func__, l, dest, src);
 198
 199     /* mvc with source pointing to the byte after the destination is the
 200        same as memset with the first source byte */
 201     if (dest == (src + 1)) {
 202         fast_memset(env, dest, cpu_ldub_data(env, src), l + 1);
 203         return;
 204     }
 205
 206     /* mvc and memmove do not behave the same when areas overlap! */
 207     if ((dest < src) || (src + l < dest)) {
 208         fast_memmove(env, dest, src, l + 1);
 209         return;
 210     }
 211
 212     /* slow version with byte accesses which always work */
 213     for (i = 0; i <= l; i++) {
 214         cpu_stb_data(env, dest + i, cpu_ldub_data(env, src + i));
 215     }
 216 }
 217
 218 /* compare unsigned byte arrays */
 219 uint32_t HELPER(clc)(CPUS390XState *env, uint32_t l, uint64_t s1, uint64_t s2)
 220 {
 221     int i;
 222     unsigned char x, y;
 223     uint32_t cc;
 224
 225     HELPER_LOG("%s l %d s1 %" PRIx64 " s2 %" PRIx64 "\n",
 226                __func__, l, s1, s2);
 227     for (i = 0; i <= l; i++) {
 228         x = cpu_ldub_data(env, s1 + i);
 229         y = cpu_ldub_data(env, s2 + i);
 230         HELPER_LOG("%02x (%c)/%02x (%c) ", x, x, y, y);
 231         if (x < y) {
 232             cc = 1;
 233             goto done;
 234         } else if (x > y) {
 235             cc = 2;
 236             goto done;
 237         }
 238     }
 239     cc = 0;
 240  done:
 241     HELPER_LOG("\n");
 242     return cc;
 243 }
 244
 245 /* compare logical under mask */
 246 uint32_t HELPER(clm)(CPUS390XState *env, uint32_t r1, uint32_t mask,
 247                      uint64_t addr)
 248 {
 249     uint8_t r, d;
 250     uint32_t cc;
 251
 252     HELPER_LOG("%s: r1 0x%x mask 0x%x addr 0x%" PRIx64 "\n", __func__, r1,
 253                mask, addr);
 254     cc = 0;
 255     while (mask) {
 256         if (mask & 8) {
 257             d = cpu_ldub_data(env, addr);
 258             r = (r1 & 0xff000000UL) >> 24;
 259             HELPER_LOG("mask 0x%x %02x/%02x (0x%" PRIx64 ") ", mask, r, d,
 260                        addr);
 261             if (r < d) {
 262                 cc = 1;
 263                 break;
 264             } else if (r > d) {
 265                 cc = 2;
 266                 break;
 267             }
 268             addr++;
 269         }
 270         mask = (mask << 1) & 0xf;
 271         r1 <<= 8;
 272     }
 273     HELPER_LOG("\n");
 274     return cc;
 275 }
 276
 277 static inline uint64_t fix_address(CPUS390XState *env, uint64_t a)
 278 {
 279     /* 31-Bit mode */
 280     if (!(env->psw.mask & PSW_MASK_64)) {
 281         a &= 0x7fffffff;
 282     }
 283     return a;
 284 }
 285
 286 static inline uint64_t get_address(CPUS390XState *env, int x2, int b2, int d2)
 287 {
 288     uint64_t r = d2;
 289     if (x2) {
 290         r += env->regs[x2];
 291     }
 292     if (b2) {
 293         r += env->regs[b2];
 294     }
 295     return fix_address(env, r);
 296 }
 297
 298 static inline uint64_t get_address_31fix(CPUS390XState *env, int reg)
 299 {
 300     return fix_address(env, env->regs[reg]);
 301 }
 302
 303 /* search string (c is byte to search, r2 is string, r1 end of string) */
 304 uint64_t HELPER(srst)(CPUS390XState *env, uint64_t r0, uint64_t end,
 305                       uint64_t str)
 306 {
 307     uint32_t len;
 308     uint8_t v, c = r0;
 309
 310     str = fix_address(env, str);
 311     end = fix_address(env, end);
 312
 313     /* Assume for now that R2 is unmodified.  */
 314     env->retxl = str;
 315
 316     /* Lest we fail to service interrupts in a timely manner, limit the
 317        amount of work we're willing to do.  For now, let's cap at 8k.  */
 318     for (len = 0; len < 0x2000; ++len) {
 319         if (str + len == end) {
 320             /* Character not found.  R1 & R2 are unmodified.  */
 321             env->cc_op = 2;
 322             return end;
 323         }
 324         v = cpu_ldub_data(env, str + len);
 325         if (v == c) {
 326             /* Character found.  Set R1 to the location; R2 is unmodified.  */
 327             env->cc_op = 1;
 328             return str + len;
 329         }
 330     }
 331
 332     /* CPU-determined bytes processed.  Advance R2 to next byte to process.  */
 333     env->retxl = str + len;
 334     env->cc_op = 3;
 335     return end;
 336 }
 337
 338 /* unsigned string compare (c is string terminator) */
 339 uint64_t HELPER(clst)(CPUS390XState *env, uint64_t c, uint64_t s1, uint64_t s2)
 340 {
 341     uint32_t len;
 342
 343     c = c & 0xff;
 344     s1 = fix_address(env, s1);
 345     s2 = fix_address(env, s2);
 346
 347     /* Lest we fail to service interrupts in a timely manner, limit the
 348        amount of work we're willing to do.  For now, let's cap at 8k.  */
 349     for (len = 0; len < 0x2000; ++len) {
 350         uint8_t v1 = cpu_ldub_data(env, s1 + len);
 351         uint8_t v2 = cpu_ldub_data(env, s2 + len);
 352         if (v1 == v2) {
 353             if (v1 == c) {
 354                 /* Equal.  CC=0, and don't advance the registers.  */
 355                 env->cc_op = 0;
 356                 env->retxl = s2;
 357                 return s1;
 358             }
 359         } else {
 360             /* Unequal.  CC={1,2}, and advance the registers.  Note that
 361                the terminator need not be zero, but the string that contains
 362                the terminator is by definition "low".  */
 363             env->cc_op = (v1 == c ? 1 : v2 == c ? 2 : v1 < v2 ? 1 : 2);
 364             env->retxl = s2 + len;
 365             return s1 + len;
 366         }
 367     }
 368
 369     /* CPU-determined bytes equal; advance the registers.  */
 370     env->cc_op = 3;
 371     env->retxl = s2 + len;
 372     return s1 + len;
 373 }
 374
 375 /* move page */
 376 void HELPER(mvpg)(CPUS390XState *env, uint64_t r0, uint64_t r1, uint64_t r2)
 377 {
 378     /* XXX missing r0 handling */
 379     env->cc_op = 0;
 380     fast_memmove(env, r1, r2, TARGET_PAGE_SIZE);
 381 }
 382
 383 /* string copy (c is string terminator) */
 384 uint64_t HELPER(mvst)(CPUS390XState *env, uint64_t c, uint64_t d, uint64_t s)
 385 {
 386     uint32_t len;
 387
 388     c = c & 0xff;
 389     d = fix_address(env, d);
 390     s = fix_address(env, s);
 391
 392     /* Lest we fail to service interrupts in a timely manner, limit the
 393        amount of work we're willing to do.  For now, let's cap at 8k.  */
 394     for (len = 0; len < 0x2000; ++len) {
 395         uint8_t v = cpu_ldub_data(env, s + len);
 396         cpu_stb_data(env, d + len, v);
 397         if (v == c) {
 398             /* Complete.  Set CC=1 and advance R1.  */
 399             env->cc_op = 1;
 400             env->retxl = s;
 401             return d + len;
 402         }
 403     }
 404
 405     /* Incomplete.  Set CC=3 and signal to advance R1 and R2.  */
 406     env->cc_op = 3;
 407     env->retxl = s + len;
 408     return d + len;
 409 }
 410
 411 static uint32_t helper_icm(CPUS390XState *env, uint32_t r1, uint64_t address,
 412                            uint32_t mask)
 413 {
 414     int pos = 24; /* top of the lower half of r1 */
 415     uint64_t rmask = 0xff000000ULL;
 416     uint8_t val = 0;
 417     int ccd = 0;
 418     uint32_t cc = 0;
 419
 420     while (mask) {
 421         if (mask & 8) {
 422             env->regs[r1] &= ~rmask;
 423             val = cpu_ldub_data(env, address);
 424             if ((val & 0x80) && !ccd) {
 425                 cc = 1;
 426             }
 427             ccd = 1;
 428             if (val && cc == 0) {
 429                 cc = 2;
 430             }
 431             env->regs[r1] |= (uint64_t)val << pos;
 432             address++;
 433         }
 434         mask = (mask << 1) & 0xf;
 435         pos -= 8;
 436         rmask >>= 8;
 437     }
 438
 439     return cc;
 440 }
 441
 442 /* execute instruction
 443    this instruction executes an insn modified with the contents of r1
 444    it does not change the executed instruction in memory
 445    it does not change the program counter
 446    in other words: tricky...
 447    currently implemented by interpreting the cases it is most commonly used in
 448 */
 449 uint32_t HELPER(ex)(CPUS390XState *env, uint32_t cc, uint64_t v1,
 450                     uint64_t addr, uint64_t ret)
 451 {
 452     S390CPU *cpu = s390_env_get_cpu(env);
 453     uint16_t insn = cpu_lduw_code(env, addr);
 454
 455     HELPER_LOG("%s: v1 0x%lx addr 0x%lx insn 0x%x\n", __func__, v1, addr,
 456                insn);
 457     if ((insn & 0xf0ff) == 0xd000) {
 458         uint32_t l, insn2, b1, b2, d1, d2;
 459
 460         l = v1 & 0xff;
 461         insn2 = cpu_ldl_code(env, addr + 2);
 462         b1 = (insn2 >> 28) & 0xf;
 463         b2 = (insn2 >> 12) & 0xf;
 464         d1 = (insn2 >> 16) & 0xfff;
 465         d2 = insn2 & 0xfff;
 466         switch (insn & 0xf00) {
 467         case 0x200:
 468             helper_mvc(env, l, get_address(env, 0, b1, d1),
 469                        get_address(env, 0, b2, d2));
 470             break;
 471         case 0x400:
 472             cc = helper_nc(env, l, get_address(env, 0, b1, d1),
 473                             get_address(env, 0, b2, d2));
 474             break;
 475         case 0x500:
 476             cc = helper_clc(env, l, get_address(env, 0, b1, d1),
 477                             get_address(env, 0, b2, d2));
 478             break;
 479         case 0x600:
 480             cc = helper_oc(env, l, get_address(env, 0, b1, d1),
 481                             get_address(env, 0, b2, d2));
 482             break;
 483         case 0x700:
 484             cc = helper_xc(env, l, get_address(env, 0, b1, d1),
 485                            get_address(env, 0, b2, d2));
 486             break;
 487         case 0xc00:
 488             helper_tr(env, l, get_address(env, 0, b1, d1),
 489                       get_address(env, 0, b2, d2));
 490             break;
 491         case 0xd00:
 492             cc = helper_trt(env, l, get_address(env, 0, b1, d1),
 493                             get_address(env, 0, b2, d2));
 494             break;
 495         default:
 496             goto abort;
 497         }
 498     } else if ((insn & 0xff00) == 0x0a00) {
 499         /* supervisor call */
 500         HELPER_LOG("%s: svc %ld via execute\n", __func__, (insn | v1) & 0xff);
 501         env->psw.addr = ret - 4;
 502         env->int_svc_code = (insn | v1) & 0xff;
 503         env->int_svc_ilen = 4;
 504         helper_exception(env, EXCP_SVC);
 505     } else if ((insn & 0xff00) == 0xbf00) {
 506         uint32_t insn2, r1, r3, b2, d2;
 507
 508         insn2 = cpu_ldl_code(env, addr + 2);
 509         r1 = (insn2 >> 20) & 0xf;
 510         r3 = (insn2 >> 16) & 0xf;
 511         b2 = (insn2 >> 12) & 0xf;
 512         d2 = insn2 & 0xfff;
 513         cc = helper_icm(env, r1, get_address(env, 0, b2, d2), r3);
 514     } else {
 515     abort:
 516         cpu_abort(CPU(cpu), "EXECUTE on instruction prefix 0x%x not implemented\n",
 517                   insn);
 518     }
 519     return cc;
 520 }
 521
 522 /* load access registers r1 to r3 from memory at a2 */
 523 void HELPER(lam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
 524 {
 525     int i;
 526
 527     for (i = r1;; i = (i + 1) % 16) {
 528         env->aregs[i] = cpu_ldl_data(env, a2);
 529         a2 += 4;
 530
 531         if (i == r3) {
 532             break;
 533         }
 534     }
 535 }
 536
 537 /* store access registers r1 to r3 in memory at a2 */
 538 void HELPER(stam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
 539 {
 540     int i;
 541
 542     for (i = r1;; i = (i + 1) % 16) {
 543         cpu_stl_data(env, a2, env->aregs[i]);
 544         a2 += 4;
 545
 546         if (i == r3) {
 547             break;
 548         }
 549     }
 550 }
 551
 552 /* move long */
 553 uint32_t HELPER(mvcl)(CPUS390XState *env, uint32_t r1, uint32_t r2)
 554 {
 555     uint64_t destlen = env->regs[r1 + 1] & 0xffffff;
 556     uint64_t dest = get_address_31fix(env, r1);
 557     uint64_t srclen = env->regs[r2 + 1] & 0xffffff;
 558     uint64_t src = get_address_31fix(env, r2);
 559     uint8_t pad = env->regs[r2 + 1] >> 24;
 560     uint8_t v;
 561     uint32_t cc;
 562
 563     if (destlen == srclen) {
 564         cc = 0;
 565     } else if (destlen < srclen) {
 566         cc = 1;
 567     } else {
 568         cc = 2;
 569     }
 570
 571     if (srclen > destlen) {
 572         srclen = destlen;
 573     }
 574
 575     for (; destlen && srclen; src++, dest++, destlen--, srclen--) {
 576         v = cpu_ldub_data(env, src);
 577         cpu_stb_data(env, dest, v);
 578     }
 579
 580     for (; destlen; dest++, destlen--) {
 581         cpu_stb_data(env, dest, pad);
 582     }
 583
 584     env->regs[r1 + 1] = destlen;
 585     /* can't use srclen here, we trunc'ed it */
 586     env->regs[r2 + 1] -= src - env->regs[r2];
 587     env->regs[r1] = dest;
 588     env->regs[r2] = src;
 589
 590     return cc;
 591 }
 592
 593 /* move long extended another memcopy insn with more bells and whistles */
 594 uint32_t HELPER(mvcle)(CPUS390XState *env, uint32_t r1, uint64_t a2,
 595                        uint32_t r3)
 596 {
 597     uint64_t destlen = env->regs[r1 + 1];
 598     uint64_t dest = env->regs[r1];
 599     uint64_t srclen = env->regs[r3 + 1];
 600     uint64_t src = env->regs[r3];
 601     uint8_t pad = a2 & 0xff;
 602     uint8_t v;
 603     uint32_t cc;
 604
 605     if (!(env->psw.mask & PSW_MASK_64)) {
 606         destlen = (uint32_t)destlen;
 607         srclen = (uint32_t)srclen;
 608         dest &= 0x7fffffff;
 609         src &= 0x7fffffff;
 610     }
 611
 612     if (destlen == srclen) {
 613         cc = 0;
 614     } else if (destlen < srclen) {
 615         cc = 1;
 616     } else {
 617         cc = 2;
 618     }
 619
 620     if (srclen > destlen) {
 621         srclen = destlen;
 622     }
 623
 624     for (; destlen && srclen; src++, dest++, destlen--, srclen--) {
 625         v = cpu_ldub_data(env, src);
 626         cpu_stb_data(env, dest, v);
 627     }
 628
 629     for (; destlen; dest++, destlen--) {
 630         cpu_stb_data(env, dest, pad);
 631     }
 632
 633     env->regs[r1 + 1] = destlen;
 634     /* can't use srclen here, we trunc'ed it */
 635     /* FIXME: 31-bit mode! */
 636     env->regs[r3 + 1] -= src - env->regs[r3];
 637     env->regs[r1] = dest;
 638     env->regs[r3] = src;
 639
 640     return cc;
 641 }
 642
 643 /* compare logical long extended memcompare insn with padding */
 644 uint32_t HELPER(clcle)(CPUS390XState *env, uint32_t r1, uint64_t a2,
 645                        uint32_t r3)
 646 {
 647     uint64_t destlen = env->regs[r1 + 1];
 648     uint64_t dest = get_address_31fix(env, r1);
 649     uint64_t srclen = env->regs[r3 + 1];
 650     uint64_t src = get_address_31fix(env, r3);
 651     uint8_t pad = a2 & 0xff;
 652     uint8_t v1 = 0, v2 = 0;
 653     uint32_t cc = 0;
 654
 655     if (!(destlen || srclen)) {
 656         return cc;
 657     }
 658
 659     if (srclen > destlen) {
 660         srclen = destlen;
 661     }
 662
 663     for (; destlen || srclen; src++, dest++, destlen--, srclen--) {
 664         v1 = srclen ? cpu_ldub_data(env, src) : pad;
 665         v2 = destlen ? cpu_ldub_data(env, dest) : pad;
 666         if (v1 != v2) {
 667             cc = (v1 < v2) ? 1 : 2;
 668             break;
 669         }
 670     }
 671
 672     env->regs[r1 + 1] = destlen;
 673     /* can't use srclen here, we trunc'ed it */
 674     env->regs[r3 + 1] -= src - env->regs[r3];
 675     env->regs[r1] = dest;
 676     env->regs[r3] = src;
 677
 678     return cc;
 679 }
 680
 681 /* checksum */
 682 uint64_t HELPER(cksm)(CPUS390XState *env, uint64_t r1,
 683                       uint64_t src, uint64_t src_len)
 684 {
 685     uint64_t max_len, len;
 686     uint64_t cksm = (uint32_t)r1;
 687
 688     /* Lest we fail to service interrupts in a timely manner, limit the
 689        amount of work we're willing to do.  For now, let's cap at 8k.  */
 690     max_len = (src_len > 0x2000 ? 0x2000 : src_len);
 691
 692     /* Process full words as available.  */
 693     for (len = 0; len + 4 <= max_len; len += 4, src += 4) {
 694         cksm += (uint32_t)cpu_ldl_data(env, src);
 695     }
 696
 697     switch (max_len - len) {
 698     case 1:
 699         cksm += cpu_ldub_data(env, src) << 24;
 700         len += 1;
 701         break;
 702     case 2:
 703         cksm += cpu_lduw_data(env, src) << 16;
 704         len += 2;
 705         break;
 706     case 3:
 707         cksm += cpu_lduw_data(env, src) << 16;
 708         cksm += cpu_ldub_data(env, src + 2) << 8;
 709         len += 3;
 710         break;
 711     }
 712
 713     /* Fold the carry from the checksum.  Note that we can see carry-out
 714        during folding more than once (but probably not more than twice).  */
 715     while (cksm > 0xffffffffull) {
 716         cksm = (uint32_t)cksm + (cksm >> 32);
 717     }
 718
 719     /* Indicate whether or not we've processed everything.  */
 720     env->cc_op = (len == src_len ? 0 : 3);
 721
 722     /* Return both cksm and processed length.  */
 723     env->retxl = cksm;
 724     return len;
 725 }
 726
 727 void HELPER(unpk)(CPUS390XState *env, uint32_t len, uint64_t dest,
 728                   uint64_t src)
 729 {
 730     int len_dest = len >> 4;
 731     int len_src = len & 0xf;
 732     uint8_t b;
 733     int second_nibble = 0;
 734
 735     dest += len_dest;
 736     src += len_src;
 737
 738     /* last byte is special, it only flips the nibbles */
 739     b = cpu_ldub_data(env, src);
 740     cpu_stb_data(env, dest, (b << 4) | (b >> 4));
 741     src--;
 742     len_src--;
 743
 744     /* now pad every nibble with 0xf0 */
 745
 746     while (len_dest > 0) {
 747         uint8_t cur_byte = 0;
 748
 749         if (len_src > 0) {
 750             cur_byte = cpu_ldub_data(env, src);
 751         }
 752
 753         len_dest--;
 754         dest--;
 755
 756         /* only advance one nibble at a time */
 757         if (second_nibble) {
 758             cur_byte >>= 4;
 759             len_src--;
 760             src--;
 761         }
 762         second_nibble = !second_nibble;
 763
 764         /* digit */
 765         cur_byte = (cur_byte & 0xf);
 766         /* zone bits */
 767         cur_byte |= 0xf0;
 768
 769         cpu_stb_data(env, dest, cur_byte);
 770     }
 771 }
 772
 773 void HELPER(tr)(CPUS390XState *env, uint32_t len, uint64_t array,
 774                 uint64_t trans)
 775 {
 776     int i;
 777
 778     for (i = 0; i <= len; i++) {
 779         uint8_t byte = cpu_ldub_data(env, array + i);
 780         uint8_t new_byte = cpu_ldub_data(env, trans + byte);
 781
 782         cpu_stb_data(env, array + i, new_byte);
 783     }
 784 }
 785
 786 uint64_t HELPER(tre)(CPUS390XState *env, uint64_t array,
 787                      uint64_t len, uint64_t trans)
 788 {
 789     uint8_t end = env->regs[0] & 0xff;
 790     uint64_t l = len;
 791     uint64_t i;
 792
 793     if (!(env->psw.mask & PSW_MASK_64)) {
 794         array &= 0x7fffffff;
 795         l = (uint32_t)l;
 796     }
 797
 798     /* Lest we fail to service interrupts in a timely manner, limit the
 799        amount of work we're willing to do.  For now, let's cap at 8k.  */
 800     if (l > 0x2000) {
 801         l = 0x2000;
 802         env->cc_op = 3;
 803     } else {
 804         env->cc_op = 0;
 805     }
 806
 807     for (i = 0; i < l; i++) {
 808         uint8_t byte, new_byte;
 809
 810         byte = cpu_ldub_data(env, array + i);
 811
 812         if (byte == end) {
 813             env->cc_op = 1;
 814             break;
 815         }
 816
 817         new_byte = cpu_ldub_data(env, trans + byte);
 818         cpu_stb_data(env, array + i, new_byte);
 819     }
 820
 821     env->retxl = len - i;
 822     return array + i;
 823 }
 824
 825 uint32_t HELPER(trt)(CPUS390XState *env, uint32_t len, uint64_t array,
 826                      uint64_t trans)
 827 {
 828     uint32_t cc = 0;
 829     int i;
 830
 831     for (i = 0; i <= len; i++) {
 832         uint8_t byte = cpu_ldub_data(env, array + i);
 833         uint8_t sbyte = cpu_ldub_data(env, trans + byte);
 834
 835         if (sbyte != 0) {
 836             env->regs[1] = array + i;
 837             env->regs[2] = (env->regs[2] & ~0xff) | sbyte;
 838             cc = (i == len) ? 2 : 1;
 839             break;
 840         }
 841     }
 842
 843     return cc;
 844 }
 845
 846 #if !defined(CONFIG_USER_ONLY)
 847 void HELPER(lctlg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
 848 {
 849     S390CPU *cpu = s390_env_get_cpu(env);
 850     bool PERchanged = false;
 851     int i;
 852     uint64_t src = a2;
 853     uint64_t val;
 854
 855     for (i = r1;; i = (i + 1) % 16) {
 856         val = cpu_ldq_data(env, src);
 857         if (env->cregs[i] != val && i >= 9 && i <= 11) {
 858             PERchanged = true;
 859         }
 860         env->cregs[i] = val;
 861         HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%" PRIx64 "\n",
 862                    i, src, env->cregs[i]);
 863         src += sizeof(uint64_t);
 864
 865         if (i == r3) {
 866             break;
 867         }
 868     }
 869
 870     if (PERchanged && env->psw.mask & PSW_MASK_PER) {
 871         s390_cpu_recompute_watchpoints(CPU(cpu));
 872     }
 873
 874     tlb_flush(CPU(cpu), 1);
 875 }
 876
 877 void HELPER(lctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
 878 {
 879     S390CPU *cpu = s390_env_get_cpu(env);
 880     bool PERchanged = false;
 881     int i;
 882     uint64_t src = a2;
 883     uint32_t val;
 884
 885     for (i = r1;; i = (i + 1) % 16) {
 886         val = cpu_ldl_data(env, src);
 887         if ((uint32_t)env->cregs[i] != val && i >= 9 && i <= 11) {
 888             PERchanged = true;
 889         }
 890         env->cregs[i] = (env->cregs[i] & 0xFFFFFFFF00000000ULL) | val;
 891         src += sizeof(uint32_t);
 892
 893         if (i == r3) {
 894             break;
 895         }
 896     }
 897
 898     if (PERchanged && env->psw.mask & PSW_MASK_PER) {
 899         s390_cpu_recompute_watchpoints(CPU(cpu));
 900     }
 901
 902     tlb_flush(CPU(cpu), 1);
 903 }
 904
 905 void HELPER(stctg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
 906 {
 907     int i;
 908     uint64_t dest = a2;
 909
 910     for (i = r1;; i = (i + 1) % 16) {
 911         cpu_stq_data(env, dest, env->cregs[i]);
 912         dest += sizeof(uint64_t);
 913
 914         if (i == r3) {
 915             break;
 916         }
 917     }
 918 }
 919
 920 void HELPER(stctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
 921 {
 922     int i;
 923     uint64_t dest = a2;
 924
 925     for (i = r1;; i = (i + 1) % 16) {
 926         cpu_stl_data(env, dest, env->cregs[i]);
 927         dest += sizeof(uint32_t);
 928
 929         if (i == r3) {
 930             break;
 931         }
 932     }
 933 }
 934
 935 uint32_t HELPER(tprot)(uint64_t a1, uint64_t a2)
 936 {
 937     /* XXX implement */
 938
 939     return 0;
 940 }
 941
 942 /* insert storage key extended */
 943 uint64_t HELPER(iske)(CPUS390XState *env, uint64_t r2)
 944 {
 945     static S390SKeysState *ss;
 946     static S390SKeysClass *skeyclass;
 947     uint64_t addr = get_address(env, 0, 0, r2);
 948     uint8_t key;
 949
 950     if (addr > ram_size) {
 951         return 0;
 952     }
 953
 954     if (unlikely(!ss)) {
 955         ss = s390_get_skeys_device();
 956         skeyclass = S390_SKEYS_GET_CLASS(ss);
 957     }
 958
 959     if (skeyclass->get_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key)) {
 960         return 0;
 961     }
 962     return key;
 963 }
 964
 965 /* set storage key extended */
 966 void HELPER(sske)(CPUS390XState *env, uint64_t r1, uint64_t r2)
 967 {
 968     static S390SKeysState *ss;
 969     static S390SKeysClass *skeyclass;
 970     uint64_t addr = get_address(env, 0, 0, r2);
 971     uint8_t key;
 972
 973     if (addr > ram_size) {
 974         return;
 975     }
 976
 977     if (unlikely(!ss)) {
 978         ss = s390_get_skeys_device();
 979         skeyclass = S390_SKEYS_GET_CLASS(ss);
 980     }
 981
 982     key = (uint8_t) r1;
 983     skeyclass->set_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key);
 984 }
 985
 986 /* reset reference bit extended */
 987 uint32_t HELPER(rrbe)(CPUS390XState *env, uint64_t r2)
 988 {
 989     static S390SKeysState *ss;
 990     static S390SKeysClass *skeyclass;
 991     uint8_t re, key;
 992
 993     if (r2 > ram_size) {
 994         return 0;
 995     }
 996
 997     if (unlikely(!ss)) {
 998         ss = s390_get_skeys_device();
 999         skeyclass = S390_SKEYS_GET_CLASS(ss);
1000     }
1001
1002     if (skeyclass->get_skeys(ss, r2 / TARGET_PAGE_SIZE, 1, &key)) {
1003         return 0;
1004     }
1005
1006     re = key & (SK_R | SK_C);
1007     key &= ~SK_R;
1008
1009     if (skeyclass->set_skeys(ss, r2 / TARGET_PAGE_SIZE, 1, &key)) {
1010         return 0;
1011     }
1012
1013     /*
1014      * cc
1015      *
1016      * 0  Reference bit zero; change bit zero
1017      * 1  Reference bit zero; change bit one
1018      * 2  Reference bit one; change bit zero
1019      * 3  Reference bit one; change bit one
1020      */
1021
1022     return re >> 1;
1023 }
1024
1025 /* compare and swap and purge */
1026 uint32_t HELPER(csp)(CPUS390XState *env, uint32_t r1, uint64_t r2)
1027 {
1028     S390CPU *cpu = s390_env_get_cpu(env);
1029     uint32_t cc;
1030     uint32_t o1 = env->regs[r1];
1031     uint64_t a2 = r2 & ~3ULL;
1032     uint32_t o2 = cpu_ldl_data(env, a2);
1033
1034     if (o1 == o2) {
1035         cpu_stl_data(env, a2, env->regs[(r1 + 1) & 15]);
1036         if (r2 & 0x3) {
1037             /* flush TLB / ALB */
1038             tlb_flush(CPU(cpu), 1);
1039         }
1040         cc = 0;
1041     } else {
1042         env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) | o2;
1043         cc = 1;
1044     }
1045
1046     return cc;
1047 }
1048
1049 uint32_t HELPER(mvcs)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2)
1050 {
1051     int cc = 0, i;
1052
1053     HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
1054                __func__, l, a1, a2);
1055
1056     if (l > 256) {
1057         /* max 256 */
1058         l = 256;
1059         cc = 3;
1060     }
1061
1062     /* XXX replace w/ memcpy */
1063     for (i = 0; i < l; i++) {
1064         cpu_stb_secondary(env, a1 + i, cpu_ldub_primary(env, a2 + i));
1065     }
1066
1067     return cc;
1068 }
1069
1070 uint32_t HELPER(mvcp)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2)
1071 {
1072     int cc = 0, i;
1073
1074     HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
1075                __func__, l, a1, a2);
1076
1077     if (l > 256) {
1078         /* max 256 */
1079         l = 256;
1080         cc = 3;
1081     }
1082
1083     /* XXX replace w/ memcpy */
1084     for (i = 0; i < l; i++) {
1085         cpu_stb_primary(env, a1 + i, cpu_ldub_secondary(env, a2 + i));
1086     }
1087
1088     return cc;
1089 }
1090
1091 /* invalidate pte */
1092 void HELPER(ipte)(CPUS390XState *env, uint64_t pte_addr, uint64_t vaddr)
1093 {
1094     CPUState *cs = CPU(s390_env_get_cpu(env));
1095     uint64_t page = vaddr & TARGET_PAGE_MASK;
1096     uint64_t pte = 0;
1097
1098     /* XXX broadcast to other CPUs */
1099
1100     /* XXX Linux is nice enough to give us the exact pte address.
1101        According to spec we'd have to find it out ourselves */
1102     /* XXX Linux is fine with overwriting the pte, the spec requires
1103        us to only set the invalid bit */
1104     stq_phys(cs->as, pte_addr, pte | _PAGE_INVALID);
1105
1106     /* XXX we exploit the fact that Linux passes the exact virtual
1107        address here - it's not obliged to! */
1108     tlb_flush_page(cs, page);
1109
1110     /* XXX 31-bit hack */
1111     if (page & 0x80000000) {
1112         tlb_flush_page(cs, page & ~0x80000000);
1113     } else {
1114         tlb_flush_page(cs, page | 0x80000000);
1115     }
1116 }
1117
1118 /* flush local tlb */
1119 void HELPER(ptlb)(CPUS390XState *env)
1120 {
1121     S390CPU *cpu = s390_env_get_cpu(env);
1122
1123     tlb_flush(CPU(cpu), 1);
1124 }
1125
1126 /* load using real address */
1127 uint64_t HELPER(lura)(CPUS390XState *env, uint64_t addr)
1128 {
1129     CPUState *cs = CPU(s390_env_get_cpu(env));
1130
1131     return (uint32_t)ldl_phys(cs->as, get_address(env, 0, 0, addr));
1132 }
1133
1134 uint64_t HELPER(lurag)(CPUS390XState *env, uint64_t addr)
1135 {
1136     CPUState *cs = CPU(s390_env_get_cpu(env));
1137
1138     return ldq_phys(cs->as, get_address(env, 0, 0, addr));
1139 }
1140
1141 /* store using real address */
1142 void HELPER(stura)(CPUS390XState *env, uint64_t addr, uint64_t v1)
1143 {
1144     CPUState *cs = CPU(s390_env_get_cpu(env));
1145
1146     stl_phys(cs->as, get_address(env, 0, 0, addr), (uint32_t)v1);
1147
1148     if ((env->psw.mask & PSW_MASK_PER) &&
1149         (env->cregs[9] & PER_CR9_EVENT_STORE) &&
1150         (env->cregs[9] & PER_CR9_EVENT_STORE_REAL)) {
1151         /* PSW is saved just before calling the helper.  */
1152         env->per_address = env->psw.addr;
1153         env->per_perc_atmid = PER_CODE_EVENT_STORE_REAL | get_per_atmid(env);
1154     }
1155 }
1156
1157 void HELPER(sturg)(CPUS390XState *env, uint64_t addr, uint64_t v1)
1158 {
1159     CPUState *cs = CPU(s390_env_get_cpu(env));
1160
1161     stq_phys(cs->as, get_address(env, 0, 0, addr), v1);
1162
1163     if ((env->psw.mask & PSW_MASK_PER) &&
1164         (env->cregs[9] & PER_CR9_EVENT_STORE) &&
1165         (env->cregs[9] & PER_CR9_EVENT_STORE_REAL)) {
1166         /* PSW is saved just before calling the helper.  */
1167         env->per_address = env->psw.addr;
1168         env->per_perc_atmid = PER_CODE_EVENT_STORE_REAL | get_per_atmid(env);
1169     }
1170 }
1171
1172 /* load real address */
1173 uint64_t HELPER(lra)(CPUS390XState *env, uint64_t addr)
1174 {
1175     CPUState *cs = CPU(s390_env_get_cpu(env));
1176     uint32_t cc = 0;
1177     int old_exc = cs->exception_index;
1178     uint64_t asc = env->psw.mask & PSW_MASK_ASC;
1179     uint64_t ret;
1180     int flags;
1181
1182     /* XXX incomplete - has more corner cases */
1183     if (!(env->psw.mask & PSW_MASK_64) && (addr >> 32)) {
1184         program_interrupt(env, PGM_SPECIAL_OP, 2);
1185     }
1186
1187     cs->exception_index = old_exc;
1188     if (mmu_translate(env, addr, 0, asc, &ret, &flags, true)) {
1189         cc = 3;
1190     }
1191     if (cs->exception_index == EXCP_PGM) {
1192         ret = env->int_pgm_code | 0x80000000;
1193     } else {
1194         ret |= addr & ~TARGET_PAGE_MASK;
1195     }
1196     cs->exception_index = old_exc;
1197
1198     env->cc_op = cc;
1199     return ret;
1200 }
1201 #endif