?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
linux-user: Remove THREAD macro
[qemu/ar7.git] / target / s390x / mem_helper.c
blob427b795a78c1bf7770f91e149a2ecc9feeef6482
1 /*
2 * S/390 memory access helper routines
3 *
4 * Copyright (c) 2009 Ulrich Hecht
5 * Copyright (c) 2009 Alexander Graf
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 */
20
21 #include "qemu/osdep.h"
22 #include "cpu.h"
23 #include "internal.h"
24 #include "exec/address-spaces.h"
25 #include "exec/helper-proto.h"
26 #include "exec/exec-all.h"
27 #include "exec/cpu_ldst.h"
28 #include "qemu/int128.h"
29
30 #if !defined(CONFIG_USER_ONLY)
31 #include "hw/s390x/storage-keys.h"
32 #endif
33
34 /*****************************************************************************/
35 /* Softmmu support */
36 #if !defined(CONFIG_USER_ONLY)
37
38 /* try to fill the TLB and return an exception if error. If retaddr is
39 NULL, it means that the function was called in C code (i.e. not
40 from generated code or from helper.c) */
41 /* XXX: fix it to restore all registers */
42 void tlb_fill(CPUState *cs, target_ulong addr, int size,
43 MMUAccessType access_type, int mmu_idx, uintptr_t retaddr)
44 {
45 int ret = s390_cpu_handle_mmu_fault(cs, addr, size, access_type, mmu_idx);
46 if (unlikely(ret != 0)) {
47 cpu_loop_exit_restore(cs, retaddr);
48 }
49 }
50
51 #endif
52
53 /* #define DEBUG_HELPER */
54 #ifdef DEBUG_HELPER
55 #define HELPER_LOG(x...) qemu_log(x)
56 #else
57 #define HELPER_LOG(x...)
58 #endif
59
60 static inline bool psw_key_valid(CPUS390XState *env, uint8_t psw_key)
61 {
62 uint16_t pkm = env->cregs[3] >> 16;
63
64 if (env->psw.mask & PSW_MASK_PSTATE) {
65 /* PSW key has range 0..15, it is valid if the bit is 1 in the PKM */
66 return pkm & (0x80 >> psw_key);
67 }
68 return true;
69 }
70
71 /* Reduce the length so that addr + len doesn't cross a page boundary. */
72 static inline uint32_t adj_len_to_page(uint32_t len, uint64_t addr)
73 {
74 #ifndef CONFIG_USER_ONLY
75 if ((addr & ~TARGET_PAGE_MASK) + len - 1 >= TARGET_PAGE_SIZE) {
76 return -(addr | TARGET_PAGE_MASK);
77 }
78 #endif
79 return len;
80 }
81
82 /* Trigger a SPECIFICATION exception if an address or a length is not
83 naturally aligned. */
84 static inline void check_alignment(CPUS390XState *env, uint64_t v,
85 int wordsize, uintptr_t ra)
86 {
87 if (v % wordsize) {
88 s390_program_interrupt(env, PGM_SPECIFICATION, 6, ra);
89 }
90 }
91
92 /* Load a value from memory according to its size. */
93 static inline uint64_t cpu_ldusize_data_ra(CPUS390XState *env, uint64_t addr,
94 int wordsize, uintptr_t ra)
95 {
96 switch (wordsize) {
97 case 1:
98 return cpu_ldub_data_ra(env, addr, ra);
99 case 2:
100 return cpu_lduw_data_ra(env, addr, ra);
101 default:
102 abort();
103 }
104 }
105
106 /* Store a to memory according to its size. */
107 static inline void cpu_stsize_data_ra(CPUS390XState *env, uint64_t addr,
108 uint64_t value, int wordsize,
109 uintptr_t ra)
110 {
111 switch (wordsize) {
112 case 1:
113 cpu_stb_data_ra(env, addr, value, ra);
114 break;
115 case 2:
116 cpu_stw_data_ra(env, addr, value, ra);
117 break;
118 default:
119 abort();
120 }
121 }
122
123 static void fast_memset(CPUS390XState *env, uint64_t dest, uint8_t byte,
124 uint32_t l, uintptr_t ra)
125 {
126 int mmu_idx = cpu_mmu_index(env, false);
127
128 while (l > 0) {
129 void *p = tlb_vaddr_to_host(env, dest, MMU_DATA_STORE, mmu_idx);
130 if (p) {
131 /* Access to the whole page in write mode granted. */
132 uint32_t l_adj = adj_len_to_page(l, dest);
133 memset(p, byte, l_adj);
134 dest += l_adj;
135 l -= l_adj;
136 } else {
137 /* We failed to get access to the whole page. The next write
138 access will likely fill the QEMU TLB for the next iteration. */
139 cpu_stb_data_ra(env, dest, byte, ra);
140 dest++;
141 l--;
142 }
143 }
144 }
145
146 #ifndef CONFIG_USER_ONLY
147 static void fast_memmove_idx(CPUS390XState *env, uint64_t dest, uint64_t src,
148 uint32_t len, int dest_idx, int src_idx,
149 uintptr_t ra)
150 {
151 TCGMemOpIdx oi_dest = make_memop_idx(MO_UB, dest_idx);
152 TCGMemOpIdx oi_src = make_memop_idx(MO_UB, src_idx);
153 uint32_t len_adj;
154 void *src_p;
155 void *dest_p;
156 uint8_t x;
157
158 while (len > 0) {
159 src = wrap_address(env, src);
160 dest = wrap_address(env, dest);
161 src_p = tlb_vaddr_to_host(env, src, MMU_DATA_LOAD, src_idx);
162 dest_p = tlb_vaddr_to_host(env, dest, MMU_DATA_STORE, dest_idx);
163
164 if (src_p && dest_p) {
165 /* Access to both whole pages granted. */
166 len_adj = adj_len_to_page(adj_len_to_page(len, src), dest);
167 memmove(dest_p, src_p, len_adj);
168 } else {
169 /* We failed to get access to one or both whole pages. The next
170 read or write access will likely fill the QEMU TLB for the
171 next iteration. */
172 len_adj = 1;
173 x = helper_ret_ldub_mmu(env, src, oi_src, ra);
174 helper_ret_stb_mmu(env, dest, x, oi_dest, ra);
175 }
176 src += len_adj;
177 dest += len_adj;
178 len -= len_adj;
179 }
180 }
181
182 static int mmu_idx_from_as(uint8_t as)
183 {
184 switch (as) {
185 case AS_PRIMARY:
186 return MMU_PRIMARY_IDX;
187 case AS_SECONDARY:
188 return MMU_SECONDARY_IDX;
189 case AS_HOME:
190 return MMU_HOME_IDX;
191 default:
192 /* FIXME AS_ACCREG */
193 g_assert_not_reached();
194 }
195 }
196
197 static void fast_memmove_as(CPUS390XState *env, uint64_t dest, uint64_t src,
198 uint32_t len, uint8_t dest_as, uint8_t src_as,
199 uintptr_t ra)
200 {
201 int src_idx = mmu_idx_from_as(src_as);
202 int dest_idx = mmu_idx_from_as(dest_as);
203
204 fast_memmove_idx(env, dest, src, len, dest_idx, src_idx, ra);
205 }
206 #endif
207
208 static void fast_memmove(CPUS390XState *env, uint64_t dest, uint64_t src,
209 uint32_t l, uintptr_t ra)
210 {
211 int mmu_idx = cpu_mmu_index(env, false);
212
213 while (l > 0) {
214 void *src_p = tlb_vaddr_to_host(env, src, MMU_DATA_LOAD, mmu_idx);
215 void *dest_p = tlb_vaddr_to_host(env, dest, MMU_DATA_STORE, mmu_idx);
216 if (src_p && dest_p) {
217 /* Access to both whole pages granted. */
218 uint32_t l_adj = adj_len_to_page(l, src);
219 l_adj = adj_len_to_page(l_adj, dest);
220 memmove(dest_p, src_p, l_adj);
221 src += l_adj;
222 dest += l_adj;
223 l -= l_adj;
224 } else {
225 /* We failed to get access to one or both whole pages. The next
226 read or write access will likely fill the QEMU TLB for the
227 next iteration. */
228 cpu_stb_data_ra(env, dest, cpu_ldub_data_ra(env, src, ra), ra);
229 src++;
230 dest++;
231 l--;
232 }
233 }
234 }
235
236 /* and on array */
237 static uint32_t do_helper_nc(CPUS390XState *env, uint32_t l, uint64_t dest,
238 uint64_t src, uintptr_t ra)
239 {
240 uint32_t i;
241 uint8_t c = 0;
242
243 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
244 __func__, l, dest, src);
245
246 for (i = 0; i <= l; i++) {
247 uint8_t x = cpu_ldub_data_ra(env, src + i, ra);
248 x &= cpu_ldub_data_ra(env, dest + i, ra);
249 c |= x;
250 cpu_stb_data_ra(env, dest + i, x, ra);
251 }
252 return c != 0;
253 }
254
255 uint32_t HELPER(nc)(CPUS390XState *env, uint32_t l, uint64_t dest,
256 uint64_t src)
257 {
258 return do_helper_nc(env, l, dest, src, GETPC());
259 }
260
261 /* xor on array */
262 static uint32_t do_helper_xc(CPUS390XState *env, uint32_t l, uint64_t dest,
263 uint64_t src, uintptr_t ra)
264 {
265 uint32_t i;
266 uint8_t c = 0;
267
268 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
269 __func__, l, dest, src);
270
271 /* xor with itself is the same as memset(0) */
272 if (src == dest) {
273 fast_memset(env, dest, 0, l + 1, ra);
274 return 0;
275 }
276
277 for (i = 0; i <= l; i++) {
278 uint8_t x = cpu_ldub_data_ra(env, src + i, ra);
279 x ^= cpu_ldub_data_ra(env, dest + i, ra);
280 c |= x;
281 cpu_stb_data_ra(env, dest + i, x, ra);
282 }
283 return c != 0;
284 }
285
286 uint32_t HELPER(xc)(CPUS390XState *env, uint32_t l, uint64_t dest,
287 uint64_t src)
288 {
289 return do_helper_xc(env, l, dest, src, GETPC());
290 }
291
292 /* or on array */
293 static uint32_t do_helper_oc(CPUS390XState *env, uint32_t l, uint64_t dest,
294 uint64_t src, uintptr_t ra)
295 {
296 uint32_t i;
297 uint8_t c = 0;
298
299 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
300 __func__, l, dest, src);
301
302 for (i = 0; i <= l; i++) {
303 uint8_t x = cpu_ldub_data_ra(env, src + i, ra);
304 x |= cpu_ldub_data_ra(env, dest + i, ra);
305 c |= x;
306 cpu_stb_data_ra(env, dest + i, x, ra);
307 }
308 return c != 0;
309 }
310
311 uint32_t HELPER(oc)(CPUS390XState *env, uint32_t l, uint64_t dest,
312 uint64_t src)
313 {
314 return do_helper_oc(env, l, dest, src, GETPC());
315 }
316
317 /* memmove */
318 static uint32_t do_helper_mvc(CPUS390XState *env, uint32_t l, uint64_t dest,
319 uint64_t src, uintptr_t ra)
320 {
321 uint32_t i;
322
323 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
324 __func__, l, dest, src);
325
326 /* mvc and memmove do not behave the same when areas overlap! */
327 /* mvc with source pointing to the byte after the destination is the
328 same as memset with the first source byte */
329 if (dest == src + 1) {
330 fast_memset(env, dest, cpu_ldub_data_ra(env, src, ra), l + 1, ra);
331 } else if (dest < src || src + l < dest) {
332 fast_memmove(env, dest, src, l + 1, ra);
333 } else {
334 /* slow version with byte accesses which always work */
335 for (i = 0; i <= l; i++) {
336 uint8_t x = cpu_ldub_data_ra(env, src + i, ra);
337 cpu_stb_data_ra(env, dest + i, x, ra);
338 }
339 }
340
341 return env->cc_op;
342 }
343
344 void HELPER(mvc)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
345 {
346 do_helper_mvc(env, l, dest, src, GETPC());
347 }
348
349 /* move inverse */
350 void HELPER(mvcin)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
351 {
352 uintptr_t ra = GETPC();
353 int i;
354
355 for (i = 0; i <= l; i++) {
356 uint8_t v = cpu_ldub_data_ra(env, src - i, ra);
357 cpu_stb_data_ra(env, dest + i, v, ra);
358 }
359 }
360
361 /* move numerics */
362 void HELPER(mvn)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
363 {
364 uintptr_t ra = GETPC();
365 int i;
366
367 for (i = 0; i <= l; i++) {
368 uint8_t v = cpu_ldub_data_ra(env, dest + i, ra) & 0xf0;
369 v |= cpu_ldub_data_ra(env, src + i, ra) & 0x0f;
370 cpu_stb_data_ra(env, dest + i, v, ra);
371 }
372 }
373
374 /* move with offset */
375 void HELPER(mvo)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
376 {
377 uintptr_t ra = GETPC();
378 int len_dest = l >> 4;
379 int len_src = l & 0xf;
380 uint8_t byte_dest, byte_src;
381 int i;
382
383 src += len_src;
384 dest += len_dest;
385
386 /* Handle rightmost byte */
387 byte_src = cpu_ldub_data_ra(env, src, ra);
388 byte_dest = cpu_ldub_data_ra(env, dest, ra);
389 byte_dest = (byte_dest & 0x0f) | (byte_src << 4);
390 cpu_stb_data_ra(env, dest, byte_dest, ra);
391
392 /* Process remaining bytes from right to left */
393 for (i = 1; i <= len_dest; i++) {
394 byte_dest = byte_src >> 4;
395 if (len_src - i >= 0) {
396 byte_src = cpu_ldub_data_ra(env, src - i, ra);
397 } else {
398 byte_src = 0;
399 }
400 byte_dest |= byte_src << 4;
401 cpu_stb_data_ra(env, dest - i, byte_dest, ra);
402 }
403 }
404
405 /* move zones */
406 void HELPER(mvz)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
407 {
408 uintptr_t ra = GETPC();
409 int i;
410
411 for (i = 0; i <= l; i++) {
412 uint8_t b = cpu_ldub_data_ra(env, dest + i, ra) & 0x0f;
413 b |= cpu_ldub_data_ra(env, src + i, ra) & 0xf0;
414 cpu_stb_data_ra(env, dest + i, b, ra);
415 }
416 }
417
418 /* compare unsigned byte arrays */
419 static uint32_t do_helper_clc(CPUS390XState *env, uint32_t l, uint64_t s1,
420 uint64_t s2, uintptr_t ra)
421 {
422 uint32_t i;
423 uint32_t cc = 0;
424
425 HELPER_LOG("%s l %d s1 %" PRIx64 " s2 %" PRIx64 "\n",
426 __func__, l, s1, s2);
427
428 for (i = 0; i <= l; i++) {
429 uint8_t x = cpu_ldub_data_ra(env, s1 + i, ra);
430 uint8_t y = cpu_ldub_data_ra(env, s2 + i, ra);
431 HELPER_LOG("%02x (%c)/%02x (%c) ", x, x, y, y);
432 if (x < y) {
433 cc = 1;
434 break;
435 } else if (x > y) {
436 cc = 2;
437 break;
438 }
439 }
440
441 HELPER_LOG("\n");
442 return cc;
443 }
444
445 uint32_t HELPER(clc)(CPUS390XState *env, uint32_t l, uint64_t s1, uint64_t s2)
446 {
447 return do_helper_clc(env, l, s1, s2, GETPC());
448 }
449
450 /* compare logical under mask */
451 uint32_t HELPER(clm)(CPUS390XState *env, uint32_t r1, uint32_t mask,
452 uint64_t addr)
453 {
454 uintptr_t ra = GETPC();
455 uint32_t cc = 0;
456
457 HELPER_LOG("%s: r1 0x%x mask 0x%x addr 0x%" PRIx64 "\n", __func__, r1,
458 mask, addr);
459
460 while (mask) {
461 if (mask & 8) {
462 uint8_t d = cpu_ldub_data_ra(env, addr, ra);
463 uint8_t r = extract32(r1, 24, 8);
464 HELPER_LOG("mask 0x%x %02x/%02x (0x%" PRIx64 ") ", mask, r, d,
465 addr);
466 if (r < d) {
467 cc = 1;
468 break;
469 } else if (r > d) {
470 cc = 2;
471 break;
472 }
473 addr++;
474 }
475 mask = (mask << 1) & 0xf;
476 r1 <<= 8;
477 }
478
479 HELPER_LOG("\n");
480 return cc;
481 }
482
483 static inline uint64_t get_address(CPUS390XState *env, int reg)
484 {
485 return wrap_address(env, env->regs[reg]);
486 }
487
488 static inline void set_address(CPUS390XState *env, int reg, uint64_t address)
489 {
490 if (env->psw.mask & PSW_MASK_64) {
491 /* 64-Bit mode */
492 env->regs[reg] = address;
493 } else {
494 if (!(env->psw.mask & PSW_MASK_32)) {
495 /* 24-Bit mode. According to the PoO it is implementation
496 dependent if bits 32-39 remain unchanged or are set to
497 zeros. Choose the former so that the function can also be
498 used for TRT. */
499 env->regs[reg] = deposit64(env->regs[reg], 0, 24, address);
500 } else {
501 /* 31-Bit mode. According to the PoO it is implementation
502 dependent if bit 32 remains unchanged or is set to zero.
503 Choose the latter so that the function can also be used for
504 TRT. */
505 address &= 0x7fffffff;
506 env->regs[reg] = deposit64(env->regs[reg], 0, 32, address);
507 }
508 }
509 }
510
511 static inline uint64_t wrap_length(CPUS390XState *env, uint64_t length)
512 {
513 if (!(env->psw.mask & PSW_MASK_64)) {
514 /* 24-Bit and 31-Bit mode */
515 length &= 0x7fffffff;
516 }
517 return length;
518 }
519
520 static inline uint64_t get_length(CPUS390XState *env, int reg)
521 {
522 return wrap_length(env, env->regs[reg]);
523 }
524
525 static inline void set_length(CPUS390XState *env, int reg, uint64_t length)
526 {
527 if (env->psw.mask & PSW_MASK_64) {
528 /* 64-Bit mode */
529 env->regs[reg] = length;
530 } else {
531 /* 24-Bit and 31-Bit mode */
532 env->regs[reg] = deposit64(env->regs[reg], 0, 32, length);
533 }
534 }
535
536 /* search string (c is byte to search, r2 is string, r1 end of string) */
537 void HELPER(srst)(CPUS390XState *env, uint32_t r1, uint32_t r2)
538 {
539 uintptr_t ra = GETPC();
540 uint64_t end, str;
541 uint32_t len;
542 uint8_t v, c = env->regs[0];
543
544 /* Bits 32-55 must contain all 0. */
545 if (env->regs[0] & 0xffffff00u) {
546 s390_program_interrupt(env, PGM_SPECIFICATION, 6, ra);
547 }
548
549 str = get_address(env, r2);
550 end = get_address(env, r1);
551
552 /* Lest we fail to service interrupts in a timely manner, limit the
553 amount of work we're willing to do. For now, let's cap at 8k. */
554 for (len = 0; len < 0x2000; ++len) {
555 if (str + len == end) {
556 /* Character not found. R1 & R2 are unmodified. */
557 env->cc_op = 2;
558 return;
559 }
560 v = cpu_ldub_data_ra(env, str + len, ra);
561 if (v == c) {
562 /* Character found. Set R1 to the location; R2 is unmodified. */
563 env->cc_op = 1;
564 set_address(env, r1, str + len);
565 return;
566 }
567 }
568
569 /* CPU-determined bytes processed. Advance R2 to next byte to process. */
570 env->cc_op = 3;
571 set_address(env, r2, str + len);
572 }
573
574 void HELPER(srstu)(CPUS390XState *env, uint32_t r1, uint32_t r2)
575 {
576 uintptr_t ra = GETPC();
577 uint32_t len;
578 uint16_t v, c = env->regs[0];
579 uint64_t end, str, adj_end;
580
581 /* Bits 32-47 of R0 must be zero. */
582 if (env->regs[0] & 0xffff0000u) {
583 s390_program_interrupt(env, PGM_SPECIFICATION, 6, ra);
584 }
585
586 str = get_address(env, r2);
587 end = get_address(env, r1);
588
589 /* If the LSB of the two addresses differ, use one extra byte. */
590 adj_end = end + ((str ^ end) & 1);
591
592 /* Lest we fail to service interrupts in a timely manner, limit the
593 amount of work we're willing to do. For now, let's cap at 8k. */
594 for (len = 0; len < 0x2000; len += 2) {
595 if (str + len == adj_end) {
596 /* End of input found. */
597 env->cc_op = 2;
598 return;
599 }
600 v = cpu_lduw_data_ra(env, str + len, ra);
601 if (v == c) {
602 /* Character found. Set R1 to the location; R2 is unmodified. */
603 env->cc_op = 1;
604 set_address(env, r1, str + len);
605 return;
606 }
607 }
608
609 /* CPU-determined bytes processed. Advance R2 to next byte to process. */
610 env->cc_op = 3;
611 set_address(env, r2, str + len);
612 }
613
614 /* unsigned string compare (c is string terminator) */
615 uint64_t HELPER(clst)(CPUS390XState *env, uint64_t c, uint64_t s1, uint64_t s2)
616 {
617 uintptr_t ra = GETPC();
618 uint32_t len;
619
620 c = c & 0xff;
621 s1 = wrap_address(env, s1);
622 s2 = wrap_address(env, s2);
623
624 /* Lest we fail to service interrupts in a timely manner, limit the
625 amount of work we're willing to do. For now, let's cap at 8k. */
626 for (len = 0; len < 0x2000; ++len) {
627 uint8_t v1 = cpu_ldub_data_ra(env, s1 + len, ra);
628 uint8_t v2 = cpu_ldub_data_ra(env, s2 + len, ra);
629 if (v1 == v2) {
630 if (v1 == c) {
631 /* Equal. CC=0, and don't advance the registers. */
632 env->cc_op = 0;
633 env->retxl = s2;
634 return s1;
635 }
636 } else {
637 /* Unequal. CC={1,2}, and advance the registers. Note that
638 the terminator need not be zero, but the string that contains
639 the terminator is by definition "low". */
640 env->cc_op = (v1 == c ? 1 : v2 == c ? 2 : v1 < v2 ? 1 : 2);
641 env->retxl = s2 + len;
642 return s1 + len;
643 }
644 }
645
646 /* CPU-determined bytes equal; advance the registers. */
647 env->cc_op = 3;
648 env->retxl = s2 + len;
649 return s1 + len;
650 }
651
652 /* move page */
653 uint32_t HELPER(mvpg)(CPUS390XState *env, uint64_t r0, uint64_t r1, uint64_t r2)
654 {
655 /* ??? missing r0 handling, which includes access keys, but more
656 importantly optional suppression of the exception! */
657 fast_memmove(env, r1, r2, TARGET_PAGE_SIZE, GETPC());
658 return 0; /* data moved */
659 }
660
661 /* string copy (c is string terminator) */
662 uint64_t HELPER(mvst)(CPUS390XState *env, uint64_t c, uint64_t d, uint64_t s)
663 {
664 uintptr_t ra = GETPC();
665 uint32_t len;
666
667 c = c & 0xff;
668 d = wrap_address(env, d);
669 s = wrap_address(env, s);
670
671 /* Lest we fail to service interrupts in a timely manner, limit the
672 amount of work we're willing to do. For now, let's cap at 8k. */
673 for (len = 0; len < 0x2000; ++len) {
674 uint8_t v = cpu_ldub_data_ra(env, s + len, ra);
675 cpu_stb_data_ra(env, d + len, v, ra);
676 if (v == c) {
677 /* Complete. Set CC=1 and advance R1. */
678 env->cc_op = 1;
679 env->retxl = s;
680 return d + len;
681 }
682 }
683
684 /* Incomplete. Set CC=3 and signal to advance R1 and R2. */
685 env->cc_op = 3;
686 env->retxl = s + len;
687 return d + len;
688 }
689
690 /* load access registers r1 to r3 from memory at a2 */
691 void HELPER(lam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
692 {
693 uintptr_t ra = GETPC();
694 int i;
695
696 for (i = r1;; i = (i + 1) % 16) {
697 env->aregs[i] = cpu_ldl_data_ra(env, a2, ra);
698 a2 += 4;
699
700 if (i == r3) {
701 break;
702 }
703 }
704 }
705
706 /* store access registers r1 to r3 in memory at a2 */
707 void HELPER(stam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
708 {
709 uintptr_t ra = GETPC();
710 int i;
711
712 for (i = r1;; i = (i + 1) % 16) {
713 cpu_stl_data_ra(env, a2, env->aregs[i], ra);
714 a2 += 4;
715
716 if (i == r3) {
717 break;
718 }
719 }
720 }
721
722 /* move long helper */
723 static inline uint32_t do_mvcl(CPUS390XState *env,
724 uint64_t *dest, uint64_t *destlen,
725 uint64_t *src, uint64_t *srclen,
726 uint16_t pad, int wordsize, uintptr_t ra)
727 {
728 uint64_t len = MIN(*srclen, *destlen);
729 uint32_t cc;
730
731 if (*destlen == *srclen) {
732 cc = 0;
733 } else if (*destlen < *srclen) {
734 cc = 1;
735 } else {
736 cc = 2;
737 }
738
739 /* Copy the src array */
740 fast_memmove(env, *dest, *src, len, ra);
741 *src += len;
742 *srclen -= len;
743 *dest += len;
744 *destlen -= len;
745
746 /* Pad the remaining area */
747 if (wordsize == 1) {
748 fast_memset(env, *dest, pad, *destlen, ra);
749 *dest += *destlen;
750 *destlen = 0;
751 } else {
752 /* If remaining length is odd, pad with odd byte first. */
753 if (*destlen & 1) {
754 cpu_stb_data_ra(env, *dest, pad & 0xff, ra);
755 *dest += 1;
756 *destlen -= 1;
757 }
758 /* The remaining length is even, pad using words. */
759 for (; *destlen; *dest += 2, *destlen -= 2) {
760 cpu_stw_data_ra(env, *dest, pad, ra);
761 }
762 }
763
764 return cc;
765 }
766
767 /* move long */
768 uint32_t HELPER(mvcl)(CPUS390XState *env, uint32_t r1, uint32_t r2)
769 {
770 uintptr_t ra = GETPC();
771 uint64_t destlen = env->regs[r1 + 1] & 0xffffff;
772 uint64_t dest = get_address(env, r1);
773 uint64_t srclen = env->regs[r2 + 1] & 0xffffff;
774 uint64_t src = get_address(env, r2);
775 uint8_t pad = env->regs[r2 + 1] >> 24;
776 uint32_t cc;
777
778 cc = do_mvcl(env, &dest, &destlen, &src, &srclen, pad, 1, ra);
779
780 env->regs[r1 + 1] = deposit64(env->regs[r1 + 1], 0, 24, destlen);
781 env->regs[r2 + 1] = deposit64(env->regs[r2 + 1], 0, 24, srclen);
782 set_address(env, r1, dest);
783 set_address(env, r2, src);
784
785 return cc;
786 }
787
788 /* move long extended */
789 uint32_t HELPER(mvcle)(CPUS390XState *env, uint32_t r1, uint64_t a2,
790 uint32_t r3)
791 {
792 uintptr_t ra = GETPC();
793 uint64_t destlen = get_length(env, r1 + 1);
794 uint64_t dest = get_address(env, r1);
795 uint64_t srclen = get_length(env, r3 + 1);
796 uint64_t src = get_address(env, r3);
797 uint8_t pad = a2;
798 uint32_t cc;
799
800 cc = do_mvcl(env, &dest, &destlen, &src, &srclen, pad, 1, ra);
801
802 set_length(env, r1 + 1, destlen);
803 set_length(env, r3 + 1, srclen);
804 set_address(env, r1, dest);
805 set_address(env, r3, src);
806
807 return cc;
808 }
809
810 /* move long unicode */
811 uint32_t HELPER(mvclu)(CPUS390XState *env, uint32_t r1, uint64_t a2,
812 uint32_t r3)
813 {
814 uintptr_t ra = GETPC();
815 uint64_t destlen = get_length(env, r1 + 1);
816 uint64_t dest = get_address(env, r1);
817 uint64_t srclen = get_length(env, r3 + 1);
818 uint64_t src = get_address(env, r3);
819 uint16_t pad = a2;
820 uint32_t cc;
821
822 cc = do_mvcl(env, &dest, &destlen, &src, &srclen, pad, 2, ra);
823
824 set_length(env, r1 + 1, destlen);
825 set_length(env, r3 + 1, srclen);
826 set_address(env, r1, dest);
827 set_address(env, r3, src);
828
829 return cc;
830 }
831
832 /* compare logical long helper */
833 static inline uint32_t do_clcl(CPUS390XState *env,
834 uint64_t *src1, uint64_t *src1len,
835 uint64_t *src3, uint64_t *src3len,
836 uint16_t pad, uint64_t limit,
837 int wordsize, uintptr_t ra)
838 {
839 uint64_t len = MAX(*src1len, *src3len);
840 uint32_t cc = 0;
841
842 check_alignment(env, *src1len | *src3len, wordsize, ra);
843
844 if (!len) {
845 return cc;
846 }
847
848 /* Lest we fail to service interrupts in a timely manner, limit the
849 amount of work we're willing to do. */
850 if (len > limit) {
851 len = limit;
852 cc = 3;
853 }
854
855 for (; len; len -= wordsize) {
856 uint16_t v1 = pad;
857 uint16_t v3 = pad;
858
859 if (*src1len) {
860 v1 = cpu_ldusize_data_ra(env, *src1, wordsize, ra);
861 }
862 if (*src3len) {
863 v3 = cpu_ldusize_data_ra(env, *src3, wordsize, ra);
864 }
865
866 if (v1 != v3) {
867 cc = (v1 < v3) ? 1 : 2;
868 break;
869 }
870
871 if (*src1len) {
872 *src1 += wordsize;
873 *src1len -= wordsize;
874 }
875 if (*src3len) {
876 *src3 += wordsize;
877 *src3len -= wordsize;
878 }
879 }
880
881 return cc;
882 }
883
884
885 /* compare logical long */
886 uint32_t HELPER(clcl)(CPUS390XState *env, uint32_t r1, uint32_t r2)
887 {
888 uintptr_t ra = GETPC();
889 uint64_t src1len = extract64(env->regs[r1 + 1], 0, 24);
890 uint64_t src1 = get_address(env, r1);
891 uint64_t src3len = extract64(env->regs[r2 + 1], 0, 24);
892 uint64_t src3 = get_address(env, r2);
893 uint8_t pad = env->regs[r2 + 1] >> 24;
894 uint32_t cc;
895
896 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, -1, 1, ra);
897
898 env->regs[r1 + 1] = deposit64(env->regs[r1 + 1], 0, 24, src1len);
899 env->regs[r2 + 1] = deposit64(env->regs[r2 + 1], 0, 24, src3len);
900 set_address(env, r1, src1);
901 set_address(env, r2, src3);
902
903 return cc;
904 }
905
906 /* compare logical long extended memcompare insn with padding */
907 uint32_t HELPER(clcle)(CPUS390XState *env, uint32_t r1, uint64_t a2,
908 uint32_t r3)
909 {
910 uintptr_t ra = GETPC();
911 uint64_t src1len = get_length(env, r1 + 1);
912 uint64_t src1 = get_address(env, r1);
913 uint64_t src3len = get_length(env, r3 + 1);
914 uint64_t src3 = get_address(env, r3);
915 uint8_t pad = a2;
916 uint32_t cc;
917
918 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, 0x2000, 1, ra);
919
920 set_length(env, r1 + 1, src1len);
921 set_length(env, r3 + 1, src3len);
922 set_address(env, r1, src1);
923 set_address(env, r3, src3);
924
925 return cc;
926 }
927
928 /* compare logical long unicode memcompare insn with padding */
929 uint32_t HELPER(clclu)(CPUS390XState *env, uint32_t r1, uint64_t a2,
930 uint32_t r3)
931 {
932 uintptr_t ra = GETPC();
933 uint64_t src1len = get_length(env, r1 + 1);
934 uint64_t src1 = get_address(env, r1);
935 uint64_t src3len = get_length(env, r3 + 1);
936 uint64_t src3 = get_address(env, r3);
937 uint16_t pad = a2;
938 uint32_t cc = 0;
939
940 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, 0x1000, 2, ra);
941
942 set_length(env, r1 + 1, src1len);
943 set_length(env, r3 + 1, src3len);
944 set_address(env, r1, src1);
945 set_address(env, r3, src3);
946
947 return cc;
948 }
949
950 /* checksum */
951 uint64_t HELPER(cksm)(CPUS390XState *env, uint64_t r1,
952 uint64_t src, uint64_t src_len)
953 {
954 uintptr_t ra = GETPC();
955 uint64_t max_len, len;
956 uint64_t cksm = (uint32_t)r1;
957
958 /* Lest we fail to service interrupts in a timely manner, limit the
959 amount of work we're willing to do. For now, let's cap at 8k. */
960 max_len = (src_len > 0x2000 ? 0x2000 : src_len);
961
962 /* Process full words as available. */
963 for (len = 0; len + 4 <= max_len; len += 4, src += 4) {
964 cksm += (uint32_t)cpu_ldl_data_ra(env, src, ra);
965 }
966
967 switch (max_len - len) {
968 case 1:
969 cksm += cpu_ldub_data_ra(env, src, ra) << 24;
970 len += 1;
971 break;
972 case 2:
973 cksm += cpu_lduw_data_ra(env, src, ra) << 16;
974 len += 2;
975 break;
976 case 3:
977 cksm += cpu_lduw_data_ra(env, src, ra) << 16;
978 cksm += cpu_ldub_data_ra(env, src + 2, ra) << 8;
979 len += 3;
980 break;
981 }
982
983 /* Fold the carry from the checksum. Note that we can see carry-out
984 during folding more than once (but probably not more than twice). */
985 while (cksm > 0xffffffffull) {
986 cksm = (uint32_t)cksm + (cksm >> 32);
987 }
988
989 /* Indicate whether or not we've processed everything. */
990 env->cc_op = (len == src_len ? 0 : 3);
991
992 /* Return both cksm and processed length. */
993 env->retxl = cksm;
994 return len;
995 }
996
997 void HELPER(pack)(CPUS390XState *env, uint32_t len, uint64_t dest, uint64_t src)
998 {
999 uintptr_t ra = GETPC();
1000 int len_dest = len >> 4;
1001 int len_src = len & 0xf;
1002 uint8_t b;
1003
1004 dest += len_dest;
1005 src += len_src;
1006
1007 /* last byte is special, it only flips the nibbles */
1008 b = cpu_ldub_data_ra(env, src, ra);
1009 cpu_stb_data_ra(env, dest, (b << 4) | (b >> 4), ra);
1010 src--;
1011 len_src--;
1012
1013 /* now pack every value */
1014 while (len_dest >= 0) {
1015 b = 0;
1016
1017 if (len_src > 0) {
1018 b = cpu_ldub_data_ra(env, src, ra) & 0x0f;
1019 src--;
1020 len_src--;
1021 }
1022 if (len_src > 0) {
1023 b |= cpu_ldub_data_ra(env, src, ra) << 4;
1024 src--;
1025 len_src--;
1026 }
1027
1028 len_dest--;
1029 dest--;
1030 cpu_stb_data_ra(env, dest, b, ra);
1031 }
1032 }
1033
1034 static inline void do_pkau(CPUS390XState *env, uint64_t dest, uint64_t src,
1035 uint32_t srclen, int ssize, uintptr_t ra)
1036 {
1037 int i;
1038 /* The destination operand is always 16 bytes long. */
1039 const int destlen = 16;
1040
1041 /* The operands are processed from right to left. */
1042 src += srclen - 1;
1043 dest += destlen - 1;
1044
1045 for (i = 0; i < destlen; i++) {
1046 uint8_t b = 0;
1047
1048 /* Start with a positive sign */
1049 if (i == 0) {
1050 b = 0xc;
1051 } else if (srclen > ssize) {
1052 b = cpu_ldub_data_ra(env, src, ra) & 0x0f;
1053 src -= ssize;
1054 srclen -= ssize;
1055 }
1056
1057 if (srclen > ssize) {
1058 b |= cpu_ldub_data_ra(env, src, ra) << 4;
1059 src -= ssize;
1060 srclen -= ssize;
1061 }
1062
1063 cpu_stb_data_ra(env, dest, b, ra);
1064 dest--;
1065 }
1066 }
1067
1068
1069 void HELPER(pka)(CPUS390XState *env, uint64_t dest, uint64_t src,
1070 uint32_t srclen)
1071 {
1072 do_pkau(env, dest, src, srclen, 1, GETPC());
1073 }
1074
1075 void HELPER(pku)(CPUS390XState *env, uint64_t dest, uint64_t src,
1076 uint32_t srclen)
1077 {
1078 do_pkau(env, dest, src, srclen, 2, GETPC());
1079 }
1080
1081 void HELPER(unpk)(CPUS390XState *env, uint32_t len, uint64_t dest,
1082 uint64_t src)
1083 {
1084 uintptr_t ra = GETPC();
1085 int len_dest = len >> 4;
1086 int len_src = len & 0xf;
1087 uint8_t b;
1088 int second_nibble = 0;
1089
1090 dest += len_dest;
1091 src += len_src;
1092
1093 /* last byte is special, it only flips the nibbles */
1094 b = cpu_ldub_data_ra(env, src, ra);
1095 cpu_stb_data_ra(env, dest, (b << 4) | (b >> 4), ra);
1096 src--;
1097 len_src--;
1098
1099 /* now pad every nibble with 0xf0 */
1100
1101 while (len_dest > 0) {
1102 uint8_t cur_byte = 0;
1103
1104 if (len_src > 0) {
1105 cur_byte = cpu_ldub_data_ra(env, src, ra);
1106 }
1107
1108 len_dest--;
1109 dest--;
1110
1111 /* only advance one nibble at a time */
1112 if (second_nibble) {
1113 cur_byte >>= 4;
1114 len_src--;
1115 src--;
1116 }
1117 second_nibble = !second_nibble;
1118
1119 /* digit */
1120 cur_byte = (cur_byte & 0xf);
1121 /* zone bits */
1122 cur_byte |= 0xf0;
1123
1124 cpu_stb_data_ra(env, dest, cur_byte, ra);
1125 }
1126 }
1127
1128 static inline uint32_t do_unpkau(CPUS390XState *env, uint64_t dest,
1129 uint32_t destlen, int dsize, uint64_t src,
1130 uintptr_t ra)
1131 {
1132 int i;
1133 uint32_t cc;
1134 uint8_t b;
1135 /* The source operand is always 16 bytes long. */
1136 const int srclen = 16;
1137
1138 /* The operands are processed from right to left. */
1139 src += srclen - 1;
1140 dest += destlen - dsize;
1141
1142 /* Check for the sign. */
1143 b = cpu_ldub_data_ra(env, src, ra);
1144 src--;
1145 switch (b & 0xf) {
1146 case 0xa:
1147 case 0xc:
1148 case 0xe ... 0xf:
1149 cc = 0; /* plus */
1150 break;
1151 case 0xb:
1152 case 0xd:
1153 cc = 1; /* minus */
1154 break;
1155 default:
1156 case 0x0 ... 0x9:
1157 cc = 3; /* invalid */
1158 break;
1159 }
1160
1161 /* Now pad every nibble with 0x30, advancing one nibble at a time. */
1162 for (i = 0; i < destlen; i += dsize) {
1163 if (i == (31 * dsize)) {
1164 /* If length is 32/64 bytes, the leftmost byte is 0. */
1165 b = 0;
1166 } else if (i % (2 * dsize)) {
1167 b = cpu_ldub_data_ra(env, src, ra);
1168 src--;
1169 } else {
1170 b >>= 4;
1171 }
1172 cpu_stsize_data_ra(env, dest, 0x30 + (b & 0xf), dsize, ra);
1173 dest -= dsize;
1174 }
1175
1176 return cc;
1177 }
1178
1179 uint32_t HELPER(unpka)(CPUS390XState *env, uint64_t dest, uint32_t destlen,
1180 uint64_t src)
1181 {
1182 return do_unpkau(env, dest, destlen, 1, src, GETPC());
1183 }
1184
1185 uint32_t HELPER(unpku)(CPUS390XState *env, uint64_t dest, uint32_t destlen,
1186 uint64_t src)
1187 {
1188 return do_unpkau(env, dest, destlen, 2, src, GETPC());
1189 }
1190
1191 uint32_t HELPER(tp)(CPUS390XState *env, uint64_t dest, uint32_t destlen)
1192 {
1193 uintptr_t ra = GETPC();
1194 uint32_t cc = 0;
1195 int i;
1196
1197 for (i = 0; i < destlen; i++) {
1198 uint8_t b = cpu_ldub_data_ra(env, dest + i, ra);
1199 /* digit */
1200 cc |= (b & 0xf0) > 0x90 ? 2 : 0;
1201
1202 if (i == (destlen - 1)) {
1203 /* sign */
1204 cc |= (b & 0xf) < 0xa ? 1 : 0;
1205 } else {
1206 /* digit */
1207 cc |= (b & 0xf) > 0x9 ? 2 : 0;
1208 }
1209 }
1210
1211 return cc;
1212 }
1213
1214 static uint32_t do_helper_tr(CPUS390XState *env, uint32_t len, uint64_t array,
1215 uint64_t trans, uintptr_t ra)
1216 {
1217 uint32_t i;
1218
1219 for (i = 0; i <= len; i++) {
1220 uint8_t byte = cpu_ldub_data_ra(env, array + i, ra);
1221 uint8_t new_byte = cpu_ldub_data_ra(env, trans + byte, ra);
1222 cpu_stb_data_ra(env, array + i, new_byte, ra);
1223 }
1224
1225 return env->cc_op;
1226 }
1227
1228 void HELPER(tr)(CPUS390XState *env, uint32_t len, uint64_t array,
1229 uint64_t trans)
1230 {
1231 do_helper_tr(env, len, array, trans, GETPC());
1232 }
1233
1234 uint64_t HELPER(tre)(CPUS390XState *env, uint64_t array,
1235 uint64_t len, uint64_t trans)
1236 {
1237 uintptr_t ra = GETPC();
1238 uint8_t end = env->regs[0] & 0xff;
1239 uint64_t l = len;
1240 uint64_t i;
1241 uint32_t cc = 0;
1242
1243 if (!(env->psw.mask & PSW_MASK_64)) {
1244 array &= 0x7fffffff;
1245 l = (uint32_t)l;
1246 }
1247
1248 /* Lest we fail to service interrupts in a timely manner, limit the
1249 amount of work we're willing to do. For now, let's cap at 8k. */
1250 if (l > 0x2000) {
1251 l = 0x2000;
1252 cc = 3;
1253 }
1254
1255 for (i = 0; i < l; i++) {
1256 uint8_t byte, new_byte;
1257
1258 byte = cpu_ldub_data_ra(env, array + i, ra);
1259
1260 if (byte == end) {
1261 cc = 1;
1262 break;
1263 }
1264
1265 new_byte = cpu_ldub_data_ra(env, trans + byte, ra);
1266 cpu_stb_data_ra(env, array + i, new_byte, ra);
1267 }
1268
1269 env->cc_op = cc;
1270 env->retxl = len - i;
1271 return array + i;
1272 }
1273
1274 static inline uint32_t do_helper_trt(CPUS390XState *env, int len,
1275 uint64_t array, uint64_t trans,
1276 int inc, uintptr_t ra)
1277 {
1278 int i;
1279
1280 for (i = 0; i <= len; i++) {
1281 uint8_t byte = cpu_ldub_data_ra(env, array + i * inc, ra);
1282 uint8_t sbyte = cpu_ldub_data_ra(env, trans + byte, ra);
1283
1284 if (sbyte != 0) {
1285 set_address(env, 1, array + i * inc);
1286 env->regs[2] = deposit64(env->regs[2], 0, 8, sbyte);
1287 return (i == len) ? 2 : 1;
1288 }
1289 }
1290
1291 return 0;
1292 }
1293
1294 uint32_t HELPER(trt)(CPUS390XState *env, uint32_t len, uint64_t array,
1295 uint64_t trans)
1296 {
1297 return do_helper_trt(env, len, array, trans, 1, GETPC());
1298 }
1299
1300 uint32_t HELPER(trtr)(CPUS390XState *env, uint32_t len, uint64_t array,
1301 uint64_t trans)
1302 {
1303 return do_helper_trt(env, len, array, trans, -1, GETPC());
1304 }
1305
1306 /* Translate one/two to one/two */
1307 uint32_t HELPER(trXX)(CPUS390XState *env, uint32_t r1, uint32_t r2,
1308 uint32_t tst, uint32_t sizes)
1309 {
1310 uintptr_t ra = GETPC();
1311 int dsize = (sizes & 1) ? 1 : 2;
1312 int ssize = (sizes & 2) ? 1 : 2;
1313 uint64_t tbl = get_address(env, 1);
1314 uint64_t dst = get_address(env, r1);
1315 uint64_t len = get_length(env, r1 + 1);
1316 uint64_t src = get_address(env, r2);
1317 uint32_t cc = 3;
1318 int i;
1319
1320 /* The lower address bits of TBL are ignored. For TROO, TROT, it's
1321 the low 3 bits (double-word aligned). For TRTO, TRTT, it's either
1322 the low 12 bits (4K, without ETF2-ENH) or 3 bits (with ETF2-ENH). */
1323 if (ssize == 2 && !s390_has_feat(S390_FEAT_ETF2_ENH)) {
1324 tbl &= -4096;
1325 } else {
1326 tbl &= -8;
1327 }
1328
1329 check_alignment(env, len, ssize, ra);
1330
1331 /* Lest we fail to service interrupts in a timely manner, */
1332 /* limit the amount of work we're willing to do. */
1333 for (i = 0; i < 0x2000; i++) {
1334 uint16_t sval = cpu_ldusize_data_ra(env, src, ssize, ra);
1335 uint64_t tble = tbl + (sval * dsize);
1336 uint16_t dval = cpu_ldusize_data_ra(env, tble, dsize, ra);
1337 if (dval == tst) {
1338 cc = 1;
1339 break;
1340 }
1341 cpu_stsize_data_ra(env, dst, dval, dsize, ra);
1342
1343 len -= ssize;
1344 src += ssize;
1345 dst += dsize;
1346
1347 if (len == 0) {
1348 cc = 0;
1349 break;
1350 }
1351 }
1352
1353 set_address(env, r1, dst);
1354 set_length(env, r1 + 1, len);
1355 set_address(env, r2, src);
1356
1357 return cc;
1358 }
1359
1360 static void do_cdsg(CPUS390XState *env, uint64_t addr,
1361 uint32_t r1, uint32_t r3, bool parallel)
1362 {
1363 uintptr_t ra = GETPC();
1364 Int128 cmpv = int128_make128(env->regs[r1 + 1], env->regs[r1]);
1365 Int128 newv = int128_make128(env->regs[r3 + 1], env->regs[r3]);
1366 Int128 oldv;
1367 bool fail;
1368
1369 if (parallel) {
1370 #ifndef CONFIG_ATOMIC128
1371 cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
1372 #else
1373 int mem_idx = cpu_mmu_index(env, false);
1374 TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
1375 oldv = helper_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
1376 fail = !int128_eq(oldv, cmpv);
1377 #endif
1378 } else {
1379 uint64_t oldh, oldl;
1380
1381 check_alignment(env, addr, 16, ra);
1382
1383 oldh = cpu_ldq_data_ra(env, addr + 0, ra);
1384 oldl = cpu_ldq_data_ra(env, addr + 8, ra);
1385
1386 oldv = int128_make128(oldl, oldh);
1387 fail = !int128_eq(oldv, cmpv);
1388 if (fail) {
1389 newv = oldv;
1390 }
1391
1392 cpu_stq_data_ra(env, addr + 0, int128_gethi(newv), ra);
1393 cpu_stq_data_ra(env, addr + 8, int128_getlo(newv), ra);
1394 }
1395
1396 env->cc_op = fail;
1397 env->regs[r1] = int128_gethi(oldv);
1398 env->regs[r1 + 1] = int128_getlo(oldv);
1399 }
1400
1401 void HELPER(cdsg)(CPUS390XState *env, uint64_t addr,
1402 uint32_t r1, uint32_t r3)
1403 {
1404 do_cdsg(env, addr, r1, r3, false);
1405 }
1406
1407 void HELPER(cdsg_parallel)(CPUS390XState *env, uint64_t addr,
1408 uint32_t r1, uint32_t r3)
1409 {
1410 do_cdsg(env, addr, r1, r3, true);
1411 }
1412
1413 static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1,
1414 uint64_t a2, bool parallel)
1415 {
1416 #if !defined(CONFIG_USER_ONLY) || defined(CONFIG_ATOMIC128)
1417 uint32_t mem_idx = cpu_mmu_index(env, false);
1418 #endif
1419 uintptr_t ra = GETPC();
1420 uint32_t fc = extract32(env->regs[0], 0, 8);
1421 uint32_t sc = extract32(env->regs[0], 8, 8);
1422 uint64_t pl = get_address(env, 1) & -16;
1423 uint64_t svh, svl;
1424 uint32_t cc;
1425
1426 /* Sanity check the function code and storage characteristic. */
1427 if (fc > 1 || sc > 3) {
1428 if (!s390_has_feat(S390_FEAT_COMPARE_AND_SWAP_AND_STORE_2)) {
1429 goto spec_exception;
1430 }
1431 if (fc > 2 || sc > 4 || (fc == 2 && (r3 & 1))) {
1432 goto spec_exception;
1433 }
1434 }
1435
1436 /* Sanity check the alignments. */
1437 if (extract32(a1, 0, 4 << fc) || extract32(a2, 0, 1 << sc)) {
1438 goto spec_exception;
1439 }
1440
1441 /* Sanity check writability of the store address. */
1442 #ifndef CONFIG_USER_ONLY
1443 probe_write(env, a2, 0, mem_idx, ra);
1444 #endif
1445
1446 /* Note that the compare-and-swap is atomic, and the store is atomic, but
1447 the complete operation is not. Therefore we do not need to assert serial
1448 context in order to implement this. That said, restart early if we can't
1449 support either operation that is supposed to be atomic. */
1450 if (parallel) {
1451 int mask = 0;
1452 #if !defined(CONFIG_ATOMIC64)
1453 mask = -8;
1454 #elif !defined(CONFIG_ATOMIC128)
1455 mask = -16;
1456 #endif
1457 if (((4 << fc) | (1 << sc)) & mask) {
1458 cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
1459 }
1460 }
1461
1462 /* All loads happen before all stores. For simplicity, load the entire
1463 store value area from the parameter list. */
1464 svh = cpu_ldq_data_ra(env, pl + 16, ra);
1465 svl = cpu_ldq_data_ra(env, pl + 24, ra);
1466
1467 switch (fc) {
1468 case 0:
1469 {
1470 uint32_t nv = cpu_ldl_data_ra(env, pl, ra);
1471 uint32_t cv = env->regs[r3];
1472 uint32_t ov;
1473
1474 if (parallel) {
1475 #ifdef CONFIG_USER_ONLY
1476 uint32_t *haddr = g2h(a1);
1477 ov = atomic_cmpxchg__nocheck(haddr, cv, nv);
1478 #else
1479 TCGMemOpIdx oi = make_memop_idx(MO_TEUL | MO_ALIGN, mem_idx);
1480 ov = helper_atomic_cmpxchgl_be_mmu(env, a1, cv, nv, oi, ra);
1481 #endif
1482 } else {
1483 ov = cpu_ldl_data_ra(env, a1, ra);
1484 cpu_stl_data_ra(env, a1, (ov == cv ? nv : ov), ra);
1485 }
1486 cc = (ov != cv);
1487 env->regs[r3] = deposit64(env->regs[r3], 32, 32, ov);
1488 }
1489 break;
1490
1491 case 1:
1492 {
1493 uint64_t nv = cpu_ldq_data_ra(env, pl, ra);
1494 uint64_t cv = env->regs[r3];
1495 uint64_t ov;
1496
1497 if (parallel) {
1498 #ifdef CONFIG_ATOMIC64
1499 # ifdef CONFIG_USER_ONLY
1500 uint64_t *haddr = g2h(a1);
1501 ov = atomic_cmpxchg__nocheck(haddr, cv, nv);
1502 # else
1503 TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN, mem_idx);
1504 ov = helper_atomic_cmpxchgq_be_mmu(env, a1, cv, nv, oi, ra);
1505 # endif
1506 #else
1507 /* Note that we asserted !parallel above. */
1508 g_assert_not_reached();
1509 #endif
1510 } else {
1511 ov = cpu_ldq_data_ra(env, a1, ra);
1512 cpu_stq_data_ra(env, a1, (ov == cv ? nv : ov), ra);
1513 }
1514 cc = (ov != cv);
1515 env->regs[r3] = ov;
1516 }
1517 break;
1518
1519 case 2:
1520 {
1521 uint64_t nvh = cpu_ldq_data_ra(env, pl, ra);
1522 uint64_t nvl = cpu_ldq_data_ra(env, pl + 8, ra);
1523 Int128 nv = int128_make128(nvl, nvh);
1524 Int128 cv = int128_make128(env->regs[r3 + 1], env->regs[r3]);
1525 Int128 ov;
1526
1527 if (parallel) {
1528 #ifdef CONFIG_ATOMIC128
1529 TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
1530 ov = helper_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi, ra);
1531 cc = !int128_eq(ov, cv);
1532 #else
1533 /* Note that we asserted !parallel above. */
1534 g_assert_not_reached();
1535 #endif
1536 } else {
1537 uint64_t oh = cpu_ldq_data_ra(env, a1 + 0, ra);
1538 uint64_t ol = cpu_ldq_data_ra(env, a1 + 8, ra);
1539
1540 ov = int128_make128(ol, oh);
1541 cc = !int128_eq(ov, cv);
1542 if (cc) {
1543 nv = ov;
1544 }
1545
1546 cpu_stq_data_ra(env, a1 + 0, int128_gethi(nv), ra);
1547 cpu_stq_data_ra(env, a1 + 8, int128_getlo(nv), ra);
1548 }
1549
1550 env->regs[r3 + 0] = int128_gethi(ov);
1551 env->regs[r3 + 1] = int128_getlo(ov);
1552 }
1553 break;
1554
1555 default:
1556 g_assert_not_reached();
1557 }
1558
1559 /* Store only if the comparison succeeded. Note that above we use a pair
1560 of 64-bit big-endian loads, so for sc < 3 we must extract the value
1561 from the most-significant bits of svh. */
1562 if (cc == 0) {
1563 switch (sc) {
1564 case 0:
1565 cpu_stb_data_ra(env, a2, svh >> 56, ra);
1566 break;
1567 case 1:
1568 cpu_stw_data_ra(env, a2, svh >> 48, ra);
1569 break;
1570 case 2:
1571 cpu_stl_data_ra(env, a2, svh >> 32, ra);
1572 break;
1573 case 3:
1574 cpu_stq_data_ra(env, a2, svh, ra);
1575 break;
1576 case 4:
1577 if (parallel) {
1578 #ifdef CONFIG_ATOMIC128
1579 TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
1580 Int128 sv = int128_make128(svl, svh);
1581 helper_atomic_sto_be_mmu(env, a2, sv, oi, ra);
1582 #else
1583 /* Note that we asserted !parallel above. */
1584 g_assert_not_reached();
1585 #endif
1586 } else {
1587 cpu_stq_data_ra(env, a2 + 0, svh, ra);
1588 cpu_stq_data_ra(env, a2 + 8, svl, ra);
1589 }
1590 break;
1591 default:
1592 g_assert_not_reached();
1593 }
1594 }
1595
1596 return cc;
1597
1598 spec_exception:
1599 s390_program_interrupt(env, PGM_SPECIFICATION, 6, ra);
1600 g_assert_not_reached();
1601 }
1602
1603 uint32_t HELPER(csst)(CPUS390XState *env, uint32_t r3, uint64_t a1, uint64_t a2)
1604 {
1605 return do_csst(env, r3, a1, a2, false);
1606 }
1607
1608 uint32_t HELPER(csst_parallel)(CPUS390XState *env, uint32_t r3, uint64_t a1,
1609 uint64_t a2)
1610 {
1611 return do_csst(env, r3, a1, a2, true);
1612 }
1613
1614 #if !defined(CONFIG_USER_ONLY)
1615 void HELPER(lctlg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1616 {
1617 uintptr_t ra = GETPC();
1618 S390CPU *cpu = s390_env_get_cpu(env);
1619 bool PERchanged = false;
1620 uint64_t src = a2;
1621 uint32_t i;
1622
1623 for (i = r1;; i = (i + 1) % 16) {
1624 uint64_t val = cpu_ldq_data_ra(env, src, ra);
1625 if (env->cregs[i] != val && i >= 9 && i <= 11) {
1626 PERchanged = true;
1627 }
1628 env->cregs[i] = val;
1629 HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%" PRIx64 "\n",
1630 i, src, val);
1631 src += sizeof(uint64_t);
1632
1633 if (i == r3) {
1634 break;
1635 }
1636 }
1637
1638 if (PERchanged && env->psw.mask & PSW_MASK_PER) {
1639 s390_cpu_recompute_watchpoints(CPU(cpu));
1640 }
1641
1642 tlb_flush(CPU(cpu));
1643 }
1644
1645 void HELPER(lctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1646 {
1647 uintptr_t ra = GETPC();
1648 S390CPU *cpu = s390_env_get_cpu(env);
1649 bool PERchanged = false;
1650 uint64_t src = a2;
1651 uint32_t i;
1652
1653 for (i = r1;; i = (i + 1) % 16) {
1654 uint32_t val = cpu_ldl_data_ra(env, src, ra);
1655 if ((uint32_t)env->cregs[i] != val && i >= 9 && i <= 11) {
1656 PERchanged = true;
1657 }
1658 env->cregs[i] = deposit64(env->cregs[i], 0, 32, val);
1659 HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%x\n", i, src, val);
1660 src += sizeof(uint32_t);
1661
1662 if (i == r3) {
1663 break;
1664 }
1665 }
1666
1667 if (PERchanged && env->psw.mask & PSW_MASK_PER) {
1668 s390_cpu_recompute_watchpoints(CPU(cpu));
1669 }
1670
1671 tlb_flush(CPU(cpu));
1672 }
1673
1674 void HELPER(stctg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1675 {
1676 uintptr_t ra = GETPC();
1677 uint64_t dest = a2;
1678 uint32_t i;
1679
1680 for (i = r1;; i = (i + 1) % 16) {
1681 cpu_stq_data_ra(env, dest, env->cregs[i], ra);
1682 dest += sizeof(uint64_t);
1683
1684 if (i == r3) {
1685 break;
1686 }
1687 }
1688 }
1689
1690 void HELPER(stctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1691 {
1692 uintptr_t ra = GETPC();
1693 uint64_t dest = a2;
1694 uint32_t i;
1695
1696 for (i = r1;; i = (i + 1) % 16) {
1697 cpu_stl_data_ra(env, dest, env->cregs[i], ra);
1698 dest += sizeof(uint32_t);
1699
1700 if (i == r3) {
1701 break;
1702 }
1703 }
1704 }
1705
1706 uint32_t HELPER(testblock)(CPUS390XState *env, uint64_t real_addr)
1707 {
1708 uintptr_t ra = GETPC();
1709 int i;
1710
1711 real_addr = wrap_address(env, real_addr) & TARGET_PAGE_MASK;
1712
1713 for (i = 0; i < TARGET_PAGE_SIZE; i += 8) {
1714 cpu_stq_real_ra(env, real_addr + i, 0, ra);
1715 }
1716
1717 return 0;
1718 }
1719
1720 uint32_t HELPER(tprot)(CPUS390XState *env, uint64_t a1, uint64_t a2)
1721 {
1722 S390CPU *cpu = s390_env_get_cpu(env);
1723 CPUState *cs = CPU(cpu);
1724
1725 /*
1726 * TODO: we currently don't handle all access protection types
1727 * (including access-list and key-controlled) as well as AR mode.
1728 */
1729 if (!s390_cpu_virt_mem_check_write(cpu, a1, 0, 1)) {
1730 /* Fetching permitted; storing permitted */
1731 return 0;
1732 }
1733
1734 if (env->int_pgm_code == PGM_PROTECTION) {
1735 /* retry if reading is possible */
1736 cs->exception_index = 0;
1737 if (!s390_cpu_virt_mem_check_read(cpu, a1, 0, 1)) {
1738 /* Fetching permitted; storing not permitted */
1739 return 1;
1740 }
1741 }
1742
1743 switch (env->int_pgm_code) {
1744 case PGM_PROTECTION:
1745 /* Fetching not permitted; storing not permitted */
1746 cs->exception_index = 0;
1747 return 2;
1748 case PGM_ADDRESSING:
1749 case PGM_TRANS_SPEC:
1750 /* exceptions forwarded to the guest */
1751 s390_cpu_virt_mem_handle_exc(cpu, GETPC());
1752 return 0;
1753 }
1754
1755 /* Translation not available */
1756 cs->exception_index = 0;
1757 return 3;
1758 }
1759
1760 /* insert storage key extended */
1761 uint64_t HELPER(iske)(CPUS390XState *env, uint64_t r2)
1762 {
1763 static S390SKeysState *ss;
1764 static S390SKeysClass *skeyclass;
1765 uint64_t addr = wrap_address(env, r2);
1766 uint8_t key;
1767
1768 if (addr > ram_size) {
1769 return 0;
1770 }
1771
1772 if (unlikely(!ss)) {
1773 ss = s390_get_skeys_device();
1774 skeyclass = S390_SKEYS_GET_CLASS(ss);
1775 }
1776
1777 if (skeyclass->get_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key)) {
1778 return 0;
1779 }
1780 return key;
1781 }
1782
1783 /* set storage key extended */
1784 void HELPER(sske)(CPUS390XState *env, uint64_t r1, uint64_t r2)
1785 {
1786 static S390SKeysState *ss;
1787 static S390SKeysClass *skeyclass;
1788 uint64_t addr = wrap_address(env, r2);
1789 uint8_t key;
1790
1791 if (addr > ram_size) {
1792 return;
1793 }
1794
1795 if (unlikely(!ss)) {
1796 ss = s390_get_skeys_device();
1797 skeyclass = S390_SKEYS_GET_CLASS(ss);
1798 }
1799
1800 key = (uint8_t) r1;
1801 skeyclass->set_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key);
1802 }
1803
1804 /* reset reference bit extended */
1805 uint32_t HELPER(rrbe)(CPUS390XState *env, uint64_t r2)
1806 {
1807 static S390SKeysState *ss;
1808 static S390SKeysClass *skeyclass;
1809 uint8_t re, key;
1810
1811 if (r2 > ram_size) {
1812 return 0;
1813 }
1814
1815 if (unlikely(!ss)) {
1816 ss = s390_get_skeys_device();
1817 skeyclass = S390_SKEYS_GET_CLASS(ss);
1818 }
1819
1820 if (skeyclass->get_skeys(ss, r2 / TARGET_PAGE_SIZE, 1, &key)) {
1821 return 0;
1822 }
1823
1824 re = key & (SK_R | SK_C);
1825 key &= ~SK_R;
1826
1827 if (skeyclass->set_skeys(ss, r2 / TARGET_PAGE_SIZE, 1, &key)) {
1828 return 0;
1829 }
1830
1831 /*
1832 * cc
1833 *
1834 * 0 Reference bit zero; change bit zero
1835 * 1 Reference bit zero; change bit one
1836 * 2 Reference bit one; change bit zero
1837 * 3 Reference bit one; change bit one
1838 */
1839
1840 return re >> 1;
1841 }
1842
1843 uint32_t HELPER(mvcs)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2)
1844 {
1845 uintptr_t ra = GETPC();
1846 int cc = 0, i;
1847
1848 HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
1849 __func__, l, a1, a2);
1850
1851 if (l > 256) {
1852 /* max 256 */
1853 l = 256;
1854 cc = 3;
1855 }
1856
1857 /* XXX replace w/ memcpy */
1858 for (i = 0; i < l; i++) {
1859 uint8_t x = cpu_ldub_primary_ra(env, a2 + i, ra);
1860 cpu_stb_secondary_ra(env, a1 + i, x, ra);
1861 }
1862
1863 return cc;
1864 }
1865
1866 uint32_t HELPER(mvcp)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2)
1867 {
1868 uintptr_t ra = GETPC();
1869 int cc = 0, i;
1870
1871 HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
1872 __func__, l, a1, a2);
1873
1874 if (l > 256) {
1875 /* max 256 */
1876 l = 256;
1877 cc = 3;
1878 }
1879
1880 /* XXX replace w/ memcpy */
1881 for (i = 0; i < l; i++) {
1882 uint8_t x = cpu_ldub_secondary_ra(env, a2 + i, ra);
1883 cpu_stb_primary_ra(env, a1 + i, x, ra);
1884 }
1885
1886 return cc;
1887 }
1888
1889 void HELPER(idte)(CPUS390XState *env, uint64_t r1, uint64_t r2, uint32_t m4)
1890 {
1891 CPUState *cs = CPU(s390_env_get_cpu(env));
1892 const uintptr_t ra = GETPC();
1893 uint64_t table, entry, raddr;
1894 uint16_t entries, i, index = 0;
1895
1896 if (r2 & 0xff000) {
1897 s390_program_interrupt(env, PGM_SPECIFICATION, 4, ra);
1898 }
1899
1900 if (!(r2 & 0x800)) {
1901 /* invalidation-and-clearing operation */
1902 table = r1 & _ASCE_ORIGIN;
1903 entries = (r2 & 0x7ff) + 1;
1904
1905 switch (r1 & _ASCE_TYPE_MASK) {
1906 case _ASCE_TYPE_REGION1:
1907 index = (r2 >> 53) & 0x7ff;
1908 break;
1909 case _ASCE_TYPE_REGION2:
1910 index = (r2 >> 42) & 0x7ff;
1911 break;
1912 case _ASCE_TYPE_REGION3:
1913 index = (r2 >> 31) & 0x7ff;
1914 break;
1915 case _ASCE_TYPE_SEGMENT:
1916 index = (r2 >> 20) & 0x7ff;
1917 break;
1918 }
1919 for (i = 0; i < entries; i++) {
1920 /* addresses are not wrapped in 24/31bit mode but table index is */
1921 raddr = table + ((index + i) & 0x7ff) * sizeof(entry);
1922 entry = cpu_ldq_real_ra(env, raddr, ra);
1923 if (!(entry & _REGION_ENTRY_INV)) {
1924 /* we are allowed to not store if already invalid */
1925 entry |= _REGION_ENTRY_INV;
1926 cpu_stq_real_ra(env, raddr, entry, ra);
1927 }
1928 }
1929 }
1930
1931 /* We simply flush the complete tlb, therefore we can ignore r3. */
1932 if (m4 & 1) {
1933 tlb_flush(cs);
1934 } else {
1935 tlb_flush_all_cpus_synced(cs);
1936 }
1937 }
1938
1939 /* invalidate pte */
1940 void HELPER(ipte)(CPUS390XState *env, uint64_t pto, uint64_t vaddr,
1941 uint32_t m4)
1942 {
1943 CPUState *cs = CPU(s390_env_get_cpu(env));
1944 const uintptr_t ra = GETPC();
1945 uint64_t page = vaddr & TARGET_PAGE_MASK;
1946 uint64_t pte_addr, pte;
1947
1948 /* Compute the page table entry address */
1949 pte_addr = (pto & _SEGMENT_ENTRY_ORIGIN);
1950 pte_addr += (vaddr & VADDR_PX) >> 9;
1951
1952 /* Mark the page table entry as invalid */
1953 pte = cpu_ldq_real_ra(env, pte_addr, ra);
1954 pte |= _PAGE_INVALID;
1955 cpu_stq_real_ra(env, pte_addr, pte, ra);
1956
1957 /* XXX we exploit the fact that Linux passes the exact virtual
1958 address here - it's not obliged to! */
1959 if (m4 & 1) {
1960 if (vaddr & ~VADDR_PX) {
1961 tlb_flush_page(cs, page);
1962 /* XXX 31-bit hack */
1963 tlb_flush_page(cs, page ^ 0x80000000);
1964 } else {
1965 /* looks like we don't have a valid virtual address */
1966 tlb_flush(cs);
1967 }
1968 } else {
1969 if (vaddr & ~VADDR_PX) {
1970 tlb_flush_page_all_cpus_synced(cs, page);
1971 /* XXX 31-bit hack */
1972 tlb_flush_page_all_cpus_synced(cs, page ^ 0x80000000);
1973 } else {
1974 /* looks like we don't have a valid virtual address */
1975 tlb_flush_all_cpus_synced(cs);
1976 }
1977 }
1978 }
1979
1980 /* flush local tlb */
1981 void HELPER(ptlb)(CPUS390XState *env)
1982 {
1983 S390CPU *cpu = s390_env_get_cpu(env);
1984
1985 tlb_flush(CPU(cpu));
1986 }
1987
1988 /* flush global tlb */
1989 void HELPER(purge)(CPUS390XState *env)
1990 {
1991 S390CPU *cpu = s390_env_get_cpu(env);
1992
1993 tlb_flush_all_cpus_synced(CPU(cpu));
1994 }
1995
1996 /* load using real address */
1997 uint64_t HELPER(lura)(CPUS390XState *env, uint64_t addr)
1998 {
1999 return cpu_ldl_real_ra(env, wrap_address(env, addr), GETPC());
2000 }
2001
2002 uint64_t HELPER(lurag)(CPUS390XState *env, uint64_t addr)
2003 {
2004 return cpu_ldq_real_ra(env, wrap_address(env, addr), GETPC());
2005 }
2006
2007 /* store using real address */
2008 void HELPER(stura)(CPUS390XState *env, uint64_t addr, uint64_t v1)
2009 {
2010 cpu_stl_real_ra(env, wrap_address(env, addr), (uint32_t)v1, GETPC());
2011
2012 if ((env->psw.mask & PSW_MASK_PER) &&
2013 (env->cregs[9] & PER_CR9_EVENT_STORE) &&
2014 (env->cregs[9] & PER_CR9_EVENT_STORE_REAL)) {
2015 /* PSW is saved just before calling the helper. */
2016 env->per_address = env->psw.addr;
2017 env->per_perc_atmid = PER_CODE_EVENT_STORE_REAL | get_per_atmid(env);
2018 }
2019 }
2020
2021 void HELPER(sturg)(CPUS390XState *env, uint64_t addr, uint64_t v1)
2022 {
2023 cpu_stq_real_ra(env, wrap_address(env, addr), v1, GETPC());
2024
2025 if ((env->psw.mask & PSW_MASK_PER) &&
2026 (env->cregs[9] & PER_CR9_EVENT_STORE) &&
2027 (env->cregs[9] & PER_CR9_EVENT_STORE_REAL)) {
2028 /* PSW is saved just before calling the helper. */
2029 env->per_address = env->psw.addr;
2030 env->per_perc_atmid = PER_CODE_EVENT_STORE_REAL | get_per_atmid(env);
2031 }
2032 }
2033
2034 /* load real address */
2035 uint64_t HELPER(lra)(CPUS390XState *env, uint64_t addr)
2036 {
2037 CPUState *cs = CPU(s390_env_get_cpu(env));
2038 uint32_t cc = 0;
2039 uint64_t asc = env->psw.mask & PSW_MASK_ASC;
2040 uint64_t ret;
2041 int old_exc, flags;
2042
2043 /* XXX incomplete - has more corner cases */
2044 if (!(env->psw.mask & PSW_MASK_64) && (addr >> 32)) {
2045 s390_program_interrupt(env, PGM_SPECIAL_OP, 2, GETPC());
2046 }
2047
2048 old_exc = cs->exception_index;
2049 if (mmu_translate(env, addr, 0, asc, &ret, &flags, true)) {
2050 cc = 3;
2051 }
2052 if (cs->exception_index == EXCP_PGM) {
2053 ret = env->int_pgm_code | 0x80000000;
2054 } else {
2055 ret |= addr & ~TARGET_PAGE_MASK;
2056 }
2057 cs->exception_index = old_exc;
2058
2059 env->cc_op = cc;
2060 return ret;
2061 }
2062 #endif
2063
2064 /* load pair from quadword */
2065 static uint64_t do_lpq(CPUS390XState *env, uint64_t addr, bool parallel)
2066 {
2067 uintptr_t ra = GETPC();
2068 uint64_t hi, lo;
2069
2070 if (parallel) {
2071 #ifndef CONFIG_ATOMIC128
2072 cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
2073 #else
2074 int mem_idx = cpu_mmu_index(env, false);
2075 TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
2076 Int128 v = helper_atomic_ldo_be_mmu(env, addr, oi, ra);
2077 hi = int128_gethi(v);
2078 lo = int128_getlo(v);
2079 #endif
2080 } else {
2081 check_alignment(env, addr, 16, ra);
2082
2083 hi = cpu_ldq_data_ra(env, addr + 0, ra);
2084 lo = cpu_ldq_data_ra(env, addr + 8, ra);
2085 }
2086
2087 env->retxl = lo;
2088 return hi;
2089 }
2090
2091 uint64_t HELPER(lpq)(CPUS390XState *env, uint64_t addr)
2092 {
2093 return do_lpq(env, addr, false);
2094 }
2095
2096 uint64_t HELPER(lpq_parallel)(CPUS390XState *env, uint64_t addr)
2097 {
2098 return do_lpq(env, addr, true);
2099 }
2100
2101 /* store pair to quadword */
2102 static void do_stpq(CPUS390XState *env, uint64_t addr,
2103 uint64_t low, uint64_t high, bool parallel)
2104 {
2105 uintptr_t ra = GETPC();
2106
2107 if (parallel) {
2108 #ifndef CONFIG_ATOMIC128
2109 cpu_loop_exit_atomic(ENV_GET_CPU(env), ra);
2110 #else
2111 int mem_idx = cpu_mmu_index(env, false);
2112 TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
2113
2114 Int128 v = int128_make128(low, high);
2115 helper_atomic_sto_be_mmu(env, addr, v, oi, ra);
2116 #endif
2117 } else {
2118 check_alignment(env, addr, 16, ra);
2119
2120 cpu_stq_data_ra(env, addr + 0, high, ra);
2121 cpu_stq_data_ra(env, addr + 8, low, ra);
2122 }
2123 }
2124
2125 void HELPER(stpq)(CPUS390XState *env, uint64_t addr,
2126 uint64_t low, uint64_t high)
2127 {
2128 do_stpq(env, addr, low, high, false);
2129 }
2130
2131 void HELPER(stpq_parallel)(CPUS390XState *env, uint64_t addr,
2132 uint64_t low, uint64_t high)
2133 {
2134 do_stpq(env, addr, low, high, true);
2135 }
2136
2137 /* Execute instruction. This instruction executes an insn modified with
2138 the contents of r1. It does not change the executed instruction in memory;
2139 it does not change the program counter.
2140
2141 Perform this by recording the modified instruction in env->ex_value.
2142 This will be noticed by cpu_get_tb_cpu_state and thus tb translation.
2143 */
2144 void HELPER(ex)(CPUS390XState *env, uint32_t ilen, uint64_t r1, uint64_t addr)
2145 {
2146 uint64_t insn = cpu_lduw_code(env, addr);
2147 uint8_t opc = insn >> 8;
2148
2149 /* Or in the contents of R1[56:63]. */
2150 insn |= r1 & 0xff;
2151
2152 /* Load the rest of the instruction. */
2153 insn <<= 48;
2154 switch (get_ilen(opc)) {
2155 case 2:
2156 break;
2157 case 4:
2158 insn |= (uint64_t)cpu_lduw_code(env, addr + 2) << 32;
2159 break;
2160 case 6:
2161 insn |= (uint64_t)(uint32_t)cpu_ldl_code(env, addr + 2) << 16;
2162 break;
2163 default:
2164 g_assert_not_reached();
2165 }
2166
2167 /* The very most common cases can be sped up by avoiding a new TB. */
2168 if ((opc & 0xf0) == 0xd0) {
2169 typedef uint32_t (*dx_helper)(CPUS390XState *, uint32_t, uint64_t,
2170 uint64_t, uintptr_t);
2171 static const dx_helper dx[16] = {
2172 [0x2] = do_helper_mvc,
2173 [0x4] = do_helper_nc,
2174 [0x5] = do_helper_clc,
2175 [0x6] = do_helper_oc,
2176 [0x7] = do_helper_xc,
2177 [0xc] = do_helper_tr,
2178 };
2179 dx_helper helper = dx[opc & 0xf];
2180
2181 if (helper) {
2182 uint32_t l = extract64(insn, 48, 8);
2183 uint32_t b1 = extract64(insn, 44, 4);
2184 uint32_t d1 = extract64(insn, 32, 12);
2185 uint32_t b2 = extract64(insn, 28, 4);
2186 uint32_t d2 = extract64(insn, 16, 12);
2187 uint64_t a1 = wrap_address(env, env->regs[b1] + d1);
2188 uint64_t a2 = wrap_address(env, env->regs[b2] + d2);
2189
2190 env->cc_op = helper(env, l, a1, a2, 0);
2191 env->psw.addr += ilen;
2192 return;
2193 }
2194 } else if (opc == 0x0a) {
2195 env->int_svc_code = extract64(insn, 48, 8);
2196 env->int_svc_ilen = ilen;
2197 helper_exception(env, EXCP_SVC);
2198 g_assert_not_reached();
2199 }
2200
2201 /* Record the insn we want to execute as well as the ilen to use
2202 during the execution of the target insn. This will also ensure
2203 that ex_value is non-zero, which flags that we are in a state
2204 that requires such execution. */
2205 env->ex_value = insn | ilen;
2206 }
2207
2208 uint32_t HELPER(mvcos)(CPUS390XState *env, uint64_t dest, uint64_t src,
2209 uint64_t len)
2210 {
2211 const uint8_t psw_key = (env->psw.mask & PSW_MASK_KEY) >> PSW_SHIFT_KEY;
2212 const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC;
2213 const uint64_t r0 = env->regs[0];
2214 const uintptr_t ra = GETPC();
2215 uint8_t dest_key, dest_as, dest_k, dest_a;
2216 uint8_t src_key, src_as, src_k, src_a;
2217 uint64_t val;
2218 int cc = 0;
2219
2220 HELPER_LOG("%s dest %" PRIx64 ", src %" PRIx64 ", len %" PRIx64 "\n",
2221 __func__, dest, src, len);
2222
2223 if (!(env->psw.mask & PSW_MASK_DAT)) {
2224 s390_program_interrupt(env, PGM_SPECIAL_OP, 6, ra);
2225 }
2226
2227 /* OAC (operand access control) for the first operand -> dest */
2228 val = (r0 & 0xffff0000ULL) >> 16;
2229 dest_key = (val >> 12) & 0xf;
2230 dest_as = (val >> 6) & 0x3;
2231 dest_k = (val >> 1) & 0x1;
2232 dest_a = val & 0x1;
2233
2234 /* OAC (operand access control) for the second operand -> src */
2235 val = (r0 & 0x0000ffffULL);
2236 src_key = (val >> 12) & 0xf;
2237 src_as = (val >> 6) & 0x3;
2238 src_k = (val >> 1) & 0x1;
2239 src_a = val & 0x1;
2240
2241 if (!dest_k) {
2242 dest_key = psw_key;
2243 }
2244 if (!src_k) {
2245 src_key = psw_key;
2246 }
2247 if (!dest_a) {
2248 dest_as = psw_as;
2249 }
2250 if (!src_a) {
2251 src_as = psw_as;
2252 }
2253
2254 if (dest_a && dest_as == AS_HOME && (env->psw.mask & PSW_MASK_PSTATE)) {
2255 s390_program_interrupt(env, PGM_SPECIAL_OP, 6, ra);
2256 }
2257 if (!(env->cregs[0] & CR0_SECONDARY) &&
2258 (dest_as == AS_SECONDARY || src_as == AS_SECONDARY)) {
2259 s390_program_interrupt(env, PGM_SPECIAL_OP, 6, ra);
2260 }
2261 if (!psw_key_valid(env, dest_key) || !psw_key_valid(env, src_key)) {
2262 s390_program_interrupt(env, PGM_PRIVILEGED, 6, ra);
2263 }
2264
2265 len = wrap_length(env, len);
2266 if (len > 4096) {
2267 cc = 3;
2268 len = 4096;
2269 }
2270
2271 /* FIXME: AR-mode and proper problem state mode (using PSW keys) missing */
2272 if (src_as == AS_ACCREG || dest_as == AS_ACCREG ||
2273 (env->psw.mask & PSW_MASK_PSTATE)) {
2274 qemu_log_mask(LOG_UNIMP, "%s: AR-mode and PSTATE support missing\n",
2275 __func__);
2276 s390_program_interrupt(env, PGM_ADDRESSING, 6, ra);
2277 }
2278
2279 /* FIXME: a) LAP
2280 * b) Access using correct keys
2281 * c) AR-mode
2282 */
2283 #ifdef CONFIG_USER_ONLY
2284 /* psw keys are never valid in user mode, we will never reach this */
2285 g_assert_not_reached();
2286 #else
2287 fast_memmove_as(env, dest, src, len, dest_as, src_as, ra);
2288 #endif
2289
2290 return cc;
2291 }
2292
2293 /* Decode a Unicode character. A return value < 0 indicates success, storing
2294 the UTF-32 result into OCHAR and the input length into OLEN. A return
2295 value >= 0 indicates failure, and the CC value to be returned. */
2296 typedef int (*decode_unicode_fn)(CPUS390XState *env, uint64_t addr,
2297 uint64_t ilen, bool enh_check, uintptr_t ra,
2298 uint32_t *ochar, uint32_t *olen);
2299
2300 /* Encode a Unicode character. A return value < 0 indicates success, storing
2301 the bytes into ADDR and the output length into OLEN. A return value >= 0
2302 indicates failure, and the CC value to be returned. */
2303 typedef int (*encode_unicode_fn)(CPUS390XState *env, uint64_t addr,
2304 uint64_t ilen, uintptr_t ra, uint32_t c,
2305 uint32_t *olen);
2306
2307 static int decode_utf8(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2308 bool enh_check, uintptr_t ra,
2309 uint32_t *ochar, uint32_t *olen)
2310 {
2311 uint8_t s0, s1, s2, s3;
2312 uint32_t c, l;
2313
2314 if (ilen < 1) {
2315 return 0;
2316 }
2317 s0 = cpu_ldub_data_ra(env, addr, ra);
2318 if (s0 <= 0x7f) {
2319 /* one byte character */
2320 l = 1;
2321 c = s0;
2322 } else if (s0 <= (enh_check ? 0xc1 : 0xbf)) {
2323 /* invalid character */
2324 return 2;
2325 } else if (s0 <= 0xdf) {
2326 /* two byte character */
2327 l = 2;
2328 if (ilen < 2) {
2329 return 0;
2330 }
2331 s1 = cpu_ldub_data_ra(env, addr + 1, ra);
2332 c = s0 & 0x1f;
2333 c = (c << 6) | (s1 & 0x3f);
2334 if (enh_check && (s1 & 0xc0) != 0x80) {
2335 return 2;
2336 }
2337 } else if (s0 <= 0xef) {
2338 /* three byte character */
2339 l = 3;
2340 if (ilen < 3) {
2341 return 0;
2342 }
2343 s1 = cpu_ldub_data_ra(env, addr + 1, ra);
2344 s2 = cpu_ldub_data_ra(env, addr + 2, ra);
2345 c = s0 & 0x0f;
2346 c = (c << 6) | (s1 & 0x3f);
2347 c = (c << 6) | (s2 & 0x3f);
2348 /* Fold the byte-by-byte range descriptions in the PoO into
2349 tests against the complete value. It disallows encodings
2350 that could be smaller, and the UTF-16 surrogates. */
2351 if (enh_check
2352 && ((s1 & 0xc0) != 0x80
2353 || (s2 & 0xc0) != 0x80
2354 || c < 0x1000
2355 || (c >= 0xd800 && c <= 0xdfff))) {
2356 return 2;
2357 }
2358 } else if (s0 <= (enh_check ? 0xf4 : 0xf7)) {
2359 /* four byte character */
2360 l = 4;
2361 if (ilen < 4) {
2362 return 0;
2363 }
2364 s1 = cpu_ldub_data_ra(env, addr + 1, ra);
2365 s2 = cpu_ldub_data_ra(env, addr + 2, ra);
2366 s3 = cpu_ldub_data_ra(env, addr + 3, ra);
2367 c = s0 & 0x07;
2368 c = (c << 6) | (s1 & 0x3f);
2369 c = (c << 6) | (s2 & 0x3f);
2370 c = (c << 6) | (s3 & 0x3f);
2371 /* See above. */
2372 if (enh_check
2373 && ((s1 & 0xc0) != 0x80
2374 || (s2 & 0xc0) != 0x80
2375 || (s3 & 0xc0) != 0x80
2376 || c < 0x010000
2377 || c > 0x10ffff)) {
2378 return 2;
2379 }
2380 } else {
2381 /* invalid character */
2382 return 2;
2383 }
2384
2385 *ochar = c;
2386 *olen = l;
2387 return -1;
2388 }
2389
2390 static int decode_utf16(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2391 bool enh_check, uintptr_t ra,
2392 uint32_t *ochar, uint32_t *olen)
2393 {
2394 uint16_t s0, s1;
2395 uint32_t c, l;
2396
2397 if (ilen < 2) {
2398 return 0;
2399 }
2400 s0 = cpu_lduw_data_ra(env, addr, ra);
2401 if ((s0 & 0xfc00) != 0xd800) {
2402 /* one word character */
2403 l = 2;
2404 c = s0;
2405 } else {
2406 /* two word character */
2407 l = 4;
2408 if (ilen < 4) {
2409 return 0;
2410 }
2411 s1 = cpu_lduw_data_ra(env, addr + 2, ra);
2412 c = extract32(s0, 6, 4) + 1;
2413 c = (c << 6) | (s0 & 0x3f);
2414 c = (c << 10) | (s1 & 0x3ff);
2415 if (enh_check && (s1 & 0xfc00) != 0xdc00) {
2416 /* invalid surrogate character */
2417 return 2;
2418 }
2419 }
2420
2421 *ochar = c;
2422 *olen = l;
2423 return -1;
2424 }
2425
2426 static int decode_utf32(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2427 bool enh_check, uintptr_t ra,
2428 uint32_t *ochar, uint32_t *olen)
2429 {
2430 uint32_t c;
2431
2432 if (ilen < 4) {
2433 return 0;
2434 }
2435 c = cpu_ldl_data_ra(env, addr, ra);
2436 if ((c >= 0xd800 && c <= 0xdbff) || c > 0x10ffff) {
2437 /* invalid unicode character */
2438 return 2;
2439 }
2440
2441 *ochar = c;
2442 *olen = 4;
2443 return -1;
2444 }
2445
2446 static int encode_utf8(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2447 uintptr_t ra, uint32_t c, uint32_t *olen)
2448 {
2449 uint8_t d[4];
2450 uint32_t l, i;
2451
2452 if (c <= 0x7f) {
2453 /* one byte character */
2454 l = 1;
2455 d[0] = c;
2456 } else if (c <= 0x7ff) {
2457 /* two byte character */
2458 l = 2;
2459 d[1] = 0x80 | extract32(c, 0, 6);
2460 d[0] = 0xc0 | extract32(c, 6, 5);
2461 } else if (c <= 0xffff) {
2462 /* three byte character */
2463 l = 3;
2464 d[2] = 0x80 | extract32(c, 0, 6);
2465 d[1] = 0x80 | extract32(c, 6, 6);
2466 d[0] = 0xe0 | extract32(c, 12, 4);
2467 } else {
2468 /* four byte character */
2469 l = 4;
2470 d[3] = 0x80 | extract32(c, 0, 6);
2471 d[2] = 0x80 | extract32(c, 6, 6);
2472 d[1] = 0x80 | extract32(c, 12, 6);
2473 d[0] = 0xf0 | extract32(c, 18, 3);
2474 }
2475
2476 if (ilen < l) {
2477 return 1;
2478 }
2479 for (i = 0; i < l; ++i) {
2480 cpu_stb_data_ra(env, addr + i, d[i], ra);
2481 }
2482
2483 *olen = l;
2484 return -1;
2485 }
2486
2487 static int encode_utf16(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2488 uintptr_t ra, uint32_t c, uint32_t *olen)
2489 {
2490 uint16_t d0, d1;
2491
2492 if (c <= 0xffff) {
2493 /* one word character */
2494 if (ilen < 2) {
2495 return 1;
2496 }
2497 cpu_stw_data_ra(env, addr, c, ra);
2498 *olen = 2;
2499 } else {
2500 /* two word character */
2501 if (ilen < 4) {
2502 return 1;
2503 }
2504 d1 = 0xdc00 | extract32(c, 0, 10);
2505 d0 = 0xd800 | extract32(c, 10, 6);
2506 d0 = deposit32(d0, 6, 4, extract32(c, 16, 5) - 1);
2507 cpu_stw_data_ra(env, addr + 0, d0, ra);
2508 cpu_stw_data_ra(env, addr + 2, d1, ra);
2509 *olen = 4;
2510 }
2511
2512 return -1;
2513 }
2514
2515 static int encode_utf32(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2516 uintptr_t ra, uint32_t c, uint32_t *olen)
2517 {
2518 if (ilen < 4) {
2519 return 1;
2520 }
2521 cpu_stl_data_ra(env, addr, c, ra);
2522 *olen = 4;
2523 return -1;
2524 }
2525
2526 static inline uint32_t convert_unicode(CPUS390XState *env, uint32_t r1,
2527 uint32_t r2, uint32_t m3, uintptr_t ra,
2528 decode_unicode_fn decode,
2529 encode_unicode_fn encode)
2530 {
2531 uint64_t dst = get_address(env, r1);
2532 uint64_t dlen = get_length(env, r1 + 1);
2533 uint64_t src = get_address(env, r2);
2534 uint64_t slen = get_length(env, r2 + 1);
2535 bool enh_check = m3 & 1;
2536 int cc, i;
2537
2538 /* Lest we fail to service interrupts in a timely manner, limit the
2539 amount of work we're willing to do. For now, let's cap at 256. */
2540 for (i = 0; i < 256; ++i) {
2541 uint32_t c, ilen, olen;
2542
2543 cc = decode(env, src, slen, enh_check, ra, &c, &ilen);
2544 if (unlikely(cc >= 0)) {
2545 break;
2546 }
2547 cc = encode(env, dst, dlen, ra, c, &olen);
2548 if (unlikely(cc >= 0)) {
2549 break;
2550 }
2551
2552 src += ilen;
2553 slen -= ilen;
2554 dst += olen;
2555 dlen -= olen;
2556 cc = 3;
2557 }
2558
2559 set_address(env, r1, dst);
2560 set_length(env, r1 + 1, dlen);
2561 set_address(env, r2, src);
2562 set_length(env, r2 + 1, slen);
2563
2564 return cc;
2565 }
2566
2567 uint32_t HELPER(cu12)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2568 {
2569 return convert_unicode(env, r1, r2, m3, GETPC(),
2570 decode_utf8, encode_utf16);
2571 }
2572
2573 uint32_t HELPER(cu14)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2574 {
2575 return convert_unicode(env, r1, r2, m3, GETPC(),
2576 decode_utf8, encode_utf32);
2577 }
2578
2579 uint32_t HELPER(cu21)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2580 {
2581 return convert_unicode(env, r1, r2, m3, GETPC(),
2582 decode_utf16, encode_utf8);
2583 }
2584
2585 uint32_t HELPER(cu24)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2586 {
2587 return convert_unicode(env, r1, r2, m3, GETPC(),
2588 decode_utf16, encode_utf32);
2589 }
2590
2591 uint32_t HELPER(cu41)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2592 {
2593 return convert_unicode(env, r1, r2, m3, GETPC(),
2594 decode_utf32, encode_utf8);
2595 }
2596
2597 uint32_t HELPER(cu42)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2598 {
2599 return convert_unicode(env, r1, r2, m3, GETPC(),
2600 decode_utf32, encode_utf16);
2601 }
AR7 emulation for QEMU